From 0ece30dc7c0e34b4c5911969b8fa99c33c6d023c Mon Sep 17 00:00:00 2001 From: George Hazan Date: Wed, 30 Nov 2022 17:48:47 +0300 Subject: Telegram: update for TDLIB --- protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt | 214 ++- .../tdlib/td/tdutils/generate/CMakeLists.txt | 20 +- .../tdutils/generate/generate_mime_types_gperf.cpp | 22 +- .../tdlib/td/tdutils/generate/mime_types.txt | 28 +- .../tdlib/td/tdutils/td/utils/AesCtrByteFlow.h | 26 +- .../tdlib/td/tdutils/td/utils/AsyncFileLog.cpp | 160 +++ .../tdlib/td/tdutils/td/utils/AsyncFileLog.h | 51 + .../tdlib/td/tdutils/td/utils/AtomicRead.h | 89 ++ .../Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp | 104 +- .../Telegram/tdlib/td/tdutils/td/utils/BigNum.h | 30 +- .../tdlib/td/tdutils/td/utils/BufferedFd.h | 62 +- .../tdlib/td/tdutils/td/utils/BufferedReader.h | 16 +- .../tdlib/td/tdutils/td/utils/BufferedUdp.cpp | 17 + .../tdlib/td/tdutils/td/utils/BufferedUdp.h | 177 +++ .../Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h | 139 +- .../tdlib/td/tdutils/td/utils/CancellationToken.h | 71 + .../tdlib/td/tdutils/td/utils/ChainScheduler.h | 376 +++++ .../tdlib/td/tdutils/td/utils/ChangesProcessor.h | 2 +- .../Telegram/tdlib/td/tdutils/td/utils/Closure.h | 84 +- .../tdlib/td/tdutils/td/utils/CombinedLog.h | 86 ++ .../td/tdutils/td/utils/ConcurrentHashTable.h | 322 +++++ .../Telegram/tdlib/td/tdutils/td/utils/Container.h | 7 +- .../Telegram/tdlib/td/tdutils/td/utils/Context.h | 44 + .../Telegram/tdlib/td/tdutils/td/utils/DecTree.h | 216 +++ .../tdlib/td/tdutils/td/utils/Destructor.h | 52 + .../tdlib/td/tdutils/td/utils/Enumerator.h | 20 +- .../tdutils/td/utils/EpochBasedMemoryReclamation.h | 201 +++ .../tdlib/td/tdutils/td/utils/ExitGuard.cpp | 20 + .../Telegram/tdlib/td/tdutils/td/utils/ExitGuard.h | 30 + .../Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp | 124 +- .../Telegram/tdlib/td/tdutils/td/utils/FileLog.h | 33 +- .../tdlib/td/tdutils/td/utils/FlatHashMap.h | 24 + .../tdlib/td/tdutils/td/utils/FlatHashMapChunks.h | 575 ++++++++ .../tdlib/td/tdutils/td/utils/FlatHashSet.h | 24 + .../tdlib/td/tdutils/td/utils/FlatHashTable.cpp | 24 + .../tdlib/td/tdutils/td/utils/FlatHashTable.h | 551 ++++++++ .../tdlib/td/tdutils/td/utils/FloodControlFast.h | 81 +- .../td/tdutils/td/utils/FloodControlGlobal.cpp | 32 + .../tdlib/td/tdutils/td/utils/FloodControlGlobal.h | 35 + .../tdlib/td/tdutils/td/utils/FloodControlStrict.h | 38 +- .../Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp | 20 - .../Telegram/tdlib/td/tdutils/td/utils/GitInfo.h | 19 - .../Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp | 59 +- .../Telegram/tdlib/td/tdutils/td/utils/Gzip.h | 20 +- .../tdlib/td/tdutils/td/utils/GzipByteFlow.cpp | 83 +- .../tdlib/td/tdutils/td/utils/GzipByteFlow.h | 7 +- .../Telegram/tdlib/td/tdutils/td/utils/Hash.h | 70 + .../Telegram/tdlib/td/tdutils/td/utils/HashMap.h | 27 + .../Telegram/tdlib/td/tdutils/td/utils/HashSet.h | 27 + .../tdlib/td/tdutils/td/utils/HashTableUtils.h | 72 + .../tdlib/td/tdutils/td/utils/HazardPointers.h | 54 +- .../Telegram/tdlib/td/tdutils/td/utils/Heap.h | 67 +- .../Telegram/tdlib/td/tdutils/td/utils/Hints.cpp | 125 +- .../Telegram/tdlib/td/tdutils/td/utils/Hints.h | 21 +- .../Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp | 134 +- .../Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h | 43 +- .../tdlib/td/tdutils/td/utils/JsonBuilder.cpp | 100 +- .../tdlib/td/tdutils/td/utils/JsonBuilder.h | 230 ++- .../Telegram/tdlib/td/tdutils/td/utils/List.h | 63 +- .../Telegram/tdlib/td/tdutils/td/utils/MapNode.h | 166 +++ .../Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h | 60 +- .../tdlib/td/tdutils/td/utils/MimeType.cpp | 2 +- .../Telegram/tdlib/td/tdutils/td/utils/MimeType.h | 2 +- .../tdlib/td/tdutils/td/utils/MovableValue.h | 16 +- .../tdlib/td/tdutils/td/utils/MpmcQueue.cpp | 15 + .../Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h | 56 +- .../tdlib/td/tdutils/td/utils/MpmcWaiter.h | 281 +++- .../tdlib/td/tdutils/td/utils/MpscLinkQueue.h | 13 +- .../tdlib/td/tdutils/td/utils/MpscPollableQueue.h | 65 +- .../Telegram/tdlib/td/tdutils/td/utils/Named.h | 2 +- .../Telegram/tdlib/td/tdutils/td/utils/NullLog.h | 19 + .../tdlib/td/tdutils/td/utils/ObjectPool.h | 14 +- .../Telegram/tdlib/td/tdutils/td/utils/Observer.h | 6 +- .../tdlib/td/tdutils/td/utils/OptionParser.cpp | 263 ++++ .../tdlib/td/tdutils/td/utils/OptionParser.h | 81 ++ .../tdlib/td/tdutils/td/utils/OptionsParser.h | 150 -- .../td/tdutils/td/utils/OrderedEventsProcessor.h | 16 +- .../Telegram/tdlib/td/tdutils/td/utils/Parser.h | 104 +- .../tdlib/td/tdutils/td/utils/PathView.cpp | 70 + .../Telegram/tdlib/td/tdutils/td/utils/PathView.h | 57 +- .../Telegram/tdlib/td/tdutils/td/utils/Promise.h | 373 +++++ .../Telegram/tdlib/td/tdutils/td/utils/Random.cpp | 119 +- .../Telegram/tdlib/td/tdutils/td/utils/Random.h | 45 +- .../tdlib/td/tdutils/td/utils/ScopeGuard.h | 15 +- .../Telegram/tdlib/td/tdutils/td/utils/SetNode.h | 129 ++ .../tdlib/td/tdutils/td/utils/SharedObjectPool.h | 52 +- .../tdlib/td/tdutils/td/utils/SharedSlice.cpp | 17 + .../tdlib/td/tdutils/td/utils/SharedSlice.h | 382 +++++ .../tdlib/td/tdutils/td/utils/Slice-decl.h | 49 +- .../Telegram/tdlib/td/tdutils/td/utils/Slice.cpp | 34 + .../Telegram/tdlib/td/tdutils/td/utils/Slice.h | 90 +- .../tdlib/td/tdutils/td/utils/SliceBuilder.h | 57 + .../Telegram/tdlib/td/tdutils/td/utils/Span.h | 158 +++ .../Telegram/tdlib/td/tdutils/td/utils/SpinLock.h | 8 +- .../tdlib/td/tdutils/td/utils/StackAllocator.cpp | 72 +- .../tdlib/td/tdutils/td/utils/StackAllocator.h | 79 +- .../Telegram/tdlib/td/tdutils/td/utils/Status.cpp | 40 +- .../Telegram/tdlib/td/tdutils/td/utils/Status.h | 253 +++- .../tdlib/td/tdutils/td/utils/StealingQueue.h | 125 ++ .../Telegram/tdlib/td/tdutils/td/utils/Storer.h | 22 +- .../tdlib/td/tdutils/td/utils/StorerBase.h | 4 +- .../tdlib/td/tdutils/td/utils/StringBuilder.cpp | 151 +- .../tdlib/td/tdutils/td/utils/StringBuilder.h | 75 +- .../tdlib/td/tdutils/td/utils/ThreadLocalStorage.h | 55 + .../tdlib/td/tdutils/td/utils/ThreadSafeCounter.h | 132 ++ .../Telegram/tdlib/td/tdutils/td/utils/Time.cpp | 36 +- .../Telegram/tdlib/td/tdutils/td/utils/Time.h | 50 +- .../Telegram/tdlib/td/tdutils/td/utils/TimedStat.h | 36 +- .../Telegram/tdlib/td/tdutils/td/utils/Timer.cpp | 41 +- .../Telegram/tdlib/td/tdutils/td/utils/Timer.h | 19 +- .../tdlib/td/tdutils/td/utils/TlDowncastHelper.h | 29 + .../tdlib/td/tdutils/td/utils/TlStorerToString.h | 180 +++ .../Telegram/tdlib/td/tdutils/td/utils/TsCerr.cpp | 64 + .../Telegram/tdlib/td/tdutils/td/utils/TsCerr.h | 33 + .../tdlib/td/tdutils/td/utils/TsFileLog.cpp | 106 ++ .../Telegram/tdlib/td/tdutils/td/utils/TsFileLog.h | 23 + .../Telegram/tdlib/td/tdutils/td/utils/TsList.h | 214 +++ .../Telegram/tdlib/td/tdutils/td/utils/TsLog.cpp | 21 + .../Telegram/tdlib/td/tdutils/td/utils/TsLog.h | 55 + .../Telegram/tdlib/td/tdutils/td/utils/UInt.h | 91 ++ .../Telegram/tdlib/td/tdutils/td/utils/Variant.h | 28 +- .../tdlib/td/tdutils/td/utils/VectorQueue.h | 94 ++ .../tdlib/td/tdutils/td/utils/WaitFreeHashMap.h | 190 +++ .../tdlib/td/tdutils/td/utils/WaitFreeHashSet.h | 141 ++ .../tdlib/td/tdutils/td/utils/WaitFreeVector.h | 69 + .../Telegram/tdlib/td/tdutils/td/utils/algorithm.h | 217 +++ protocols/Telegram/tdlib/td/tdutils/td/utils/as.h | 81 ++ .../Telegram/tdlib/td/tdutils/td/utils/base64.cpp | 289 ++-- .../Telegram/tdlib/td/tdutils/td/utils/base64.h | 11 +- .../Telegram/tdlib/td/tdutils/td/utils/benchmark.h | 18 +- .../Telegram/tdlib/td/tdutils/td/utils/bits.h | 310 ++++ .../Telegram/tdlib/td/tdutils/td/utils/buffer.cpp | 118 +- .../Telegram/tdlib/td/tdutils/td/utils/buffer.h | 206 ++- .../Telegram/tdlib/td/tdutils/td/utils/check.cpp | 23 + .../Telegram/tdlib/td/tdutils/td/utils/check.h | 32 + .../Telegram/tdlib/td/tdutils/td/utils/common.h | 45 +- .../Telegram/tdlib/td/tdutils/td/utils/config.h | 5 + .../Telegram/tdlib/td/tdutils/td/utils/config.h.in | 5 + .../Telegram/tdlib/td/tdutils/td/utils/crypto.cpp | 1115 +++++++++++++-- .../Telegram/tdlib/td/tdutils/td/utils/crypto.h | 143 +- .../Telegram/tdlib/td/tdutils/td/utils/emoji.cpp | 302 ++++ .../Telegram/tdlib/td/tdutils/td/utils/emoji.h | 32 + .../tdlib/td/tdutils/td/utils/filesystem.cpp | 96 +- .../tdlib/td/tdutils/td/utils/filesystem.h | 22 +- .../tdlib/td/tdutils/td/utils/find_boundary.cpp | 8 +- .../tdlib/td/tdutils/td/utils/find_boundary.h | 2 +- .../tdlib/td/tdutils/td/utils/fixed_vector.h | 79 ++ .../Telegram/tdlib/td/tdutils/td/utils/format.h | 68 +- .../Telegram/tdlib/td/tdutils/td/utils/int_types.h | 22 +- .../Telegram/tdlib/td/tdutils/td/utils/invoke.h | 72 +- .../Telegram/tdlib/td/tdutils/td/utils/logging.cpp | 271 ++-- .../Telegram/tdlib/td/tdutils/td/utils/logging.h | 244 ++-- .../Telegram/tdlib/td/tdutils/td/utils/misc.cpp | 200 ++- .../Telegram/tdlib/td/tdutils/td/utils/misc.h | 179 +-- .../Telegram/tdlib/td/tdutils/td/utils/optional.h | 71 +- .../tdlib/td/tdutils/td/utils/overloaded.h | 4 +- .../tdlib/td/tdutils/td/utils/port/Clocks.cpp | 81 +- .../tdlib/td/tdutils/td/utils/port/Clocks.h | 19 +- .../tdlib/td/tdutils/td/utils/port/CxCli.h | 65 +- .../tdlib/td/tdutils/td/utils/port/EventFd.h | 2 +- .../tdlib/td/tdutils/td/utils/port/EventFdBase.h | 8 +- .../Telegram/tdlib/td/tdutils/td/utils/port/Fd.cpp | 1104 --------------- .../Telegram/tdlib/td/tdutils/td/utils/port/Fd.h | 226 --- .../tdlib/td/tdutils/td/utils/port/FileFd.cpp | 481 +++++-- .../tdlib/td/tdutils/td/utils/port/FileFd.h | 50 +- .../tdlib/td/tdutils/td/utils/port/FromApp.h | 100 ++ .../tdlib/td/tdutils/td/utils/port/IPAddress.cpp | 473 +++++-- .../tdlib/td/tdutils/td/utils/port/IPAddress.h | 52 +- .../tdlib/td/tdutils/td/utils/port/IoSlice.h | 42 + .../td/tdutils/td/utils/port/MemoryMapping.cpp | 109 ++ .../tdlib/td/tdutils/td/utils/port/MemoryMapping.h | 52 + .../tdlib/td/tdutils/td/utils/port/Mutex.h | 30 + .../Telegram/tdlib/td/tdutils/td/utils/port/Poll.h | 2 +- .../tdlib/td/tdutils/td/utils/port/PollBase.h | 11 +- .../tdlib/td/tdutils/td/utils/port/PollFlags.cpp | 71 + .../tdlib/td/tdutils/td/utils/port/PollFlags.h | 122 ++ .../tdlib/td/tdutils/td/utils/port/RwMutex.h | 10 +- .../td/tdutils/td/utils/port/ServerSocketFd.cpp | 384 +++-- .../td/tdutils/td/utils/port/ServerSocketFd.h | 35 +- .../tdlib/td/tdutils/td/utils/port/SocketFd.cpp | 683 ++++++++- .../tdlib/td/tdutils/td/utils/port/SocketFd.h | 53 +- .../tdlib/td/tdutils/td/utils/port/Stat.cpp | 142 +- .../Telegram/tdlib/td/tdutils/td/utils/port/Stat.h | 26 +- .../tdlib/td/tdutils/td/utils/port/StdStreams.cpp | 251 ++++ .../tdlib/td/tdutils/td/utils/port/StdStreams.h | 49 + .../tdlib/td/tdutils/td/utils/port/UdpSocketFd.cpp | 873 ++++++++++++ .../tdlib/td/tdutils/td/utils/port/UdpSocketFd.h | 93 ++ .../tdlib/td/tdutils/td/utils/port/config.h | 21 +- .../td/tdutils/td/utils/port/detail/Epoll.cpp | 82 +- .../tdlib/td/tdutils/td/utils/port/detail/Epoll.h | 30 +- .../td/tdutils/td/utils/port/detail/EventFdBsd.cpp | 45 +- .../td/tdutils/td/utils/port/detail/EventFdBsd.h | 26 +- .../tdutils/td/utils/port/detail/EventFdLinux.cpp | 94 +- .../td/tdutils/td/utils/port/detail/EventFdLinux.h | 29 +- .../td/utils/port/detail/EventFdWindows.cpp | 41 +- .../tdutils/td/utils/port/detail/EventFdWindows.h | 24 +- .../tdlib/td/tdutils/td/utils/port/detail/Iocp.cpp | 110 ++ .../tdlib/td/tdutils/td/utils/port/detail/Iocp.h | 71 + .../td/tdutils/td/utils/port/detail/KQueue.cpp | 115 +- .../tdlib/td/tdutils/td/utils/port/detail/KQueue.h | 38 +- .../td/tdutils/td/utils/port/detail/NativeFd.cpp | 261 ++++ .../td/tdutils/td/utils/port/detail/NativeFd.h | 68 + .../tdlib/td/tdutils/td/utils/port/detail/Poll.cpp | 44 +- .../tdlib/td/tdutils/td/utils/port/detail/Poll.h | 24 +- .../td/tdutils/td/utils/port/detail/PollableFd.h | 230 +++ .../td/tdutils/td/utils/port/detail/Select.cpp | 49 +- .../tdlib/td/tdutils/td/utils/port/detail/Select.h | 27 +- .../tdutils/td/utils/port/detail/ThreadIdGuard.cpp | 34 +- .../tdutils/td/utils/port/detail/ThreadIdGuard.h | 2 +- .../tdutils/td/utils/port/detail/ThreadPthread.cpp | 217 +++ .../tdutils/td/utils/port/detail/ThreadPthread.h | 83 +- .../td/tdutils/td/utils/port/detail/ThreadStl.h | 99 +- .../tdutils/td/utils/port/detail/WineventPoll.cpp | 70 +- .../td/tdutils/td/utils/port/detail/WineventPoll.h | 28 +- .../td/tdutils/td/utils/port/detail/skip_eintr.h | 63 + .../tdlib/td/tdutils/td/utils/port/path.cpp | 304 +++- .../Telegram/tdlib/td/tdutils/td/utils/port/path.h | 223 +-- .../tdlib/td/tdutils/td/utils/port/platform.cpp | 25 + .../tdlib/td/tdutils/td/utils/port/platform.h | 29 +- .../tdlib/td/tdutils/td/utils/port/rlimit.cpp | 97 ++ .../tdlib/td/tdutils/td/utils/port/rlimit.h | 20 + .../tdlib/td/tdutils/td/utils/port/signals.cpp | 88 +- .../tdlib/td/tdutils/td/utils/port/signals.h | 6 +- .../tdlib/td/tdutils/td/utils/port/sleep.cpp | 2 +- .../tdlib/td/tdutils/td/utils/port/sleep.h | 2 +- .../tdlib/td/tdutils/td/utils/port/stacktrace.cpp | 139 ++ .../tdlib/td/tdutils/td/utils/port/stacktrace.h | 23 + .../tdlib/td/tdutils/td/utils/port/thread.h | 5 +- .../td/tdutils/td/utils/port/thread_local.cpp | 12 +- .../tdlib/td/tdutils/td/utils/port/thread_local.h | 18 +- .../tdlib/td/tdutils/td/utils/port/uname.cpp | 289 ++++ .../tdlib/td/tdutils/td/utils/port/uname.h | 15 + .../tdlib/td/tdutils/td/utils/port/user.cpp | 57 + .../Telegram/tdlib/td/tdutils/td/utils/port/user.h | 16 + .../td/tdutils/td/utils/port/wstring_convert.cpp | 111 +- .../td/tdutils/td/utils/port/wstring_convert.h | 4 +- .../Telegram/tdlib/td/tdutils/td/utils/queue.h | 40 +- .../Telegram/tdlib/td/tdutils/td/utils/tests.cpp | 271 ++++ .../Telegram/tdlib/td/tdutils/td/utils/tests.h | 258 ++-- .../tdlib/td/tdutils/td/utils/tl_helpers.h | 148 +- .../tdlib/td/tdutils/td/utils/tl_parsers.cpp | 38 +- .../tdlib/td/tdutils/td/utils/tl_parsers.h | 98 +- .../tdlib/td/tdutils/td/utils/tl_storers.h | 211 +-- .../tdlib/td/tdutils/td/utils/translit.cpp | 115 ++ .../Telegram/tdlib/td/tdutils/td/utils/translit.h | 16 + .../tdlib/td/tdutils/td/utils/type_traits.h | 25 +- .../Telegram/tdlib/td/tdutils/td/utils/uint128.h | 293 ++++ .../Telegram/tdlib/td/tdutils/td/utils/unicode.cpp | 1484 +++++++++++++++----- .../Telegram/tdlib/td/tdutils/td/utils/unicode.h | 9 +- .../tdlib/td/tdutils/td/utils/unique_ptr.h | 106 ++ .../Telegram/tdlib/td/tdutils/td/utils/utf8.cpp | 117 +- .../Telegram/tdlib/td/tdutils/td/utils/utf8.h | 48 +- .../tdlib/td/tdutils/test/ChainScheduler.cpp | 244 ++++ .../tdlib/td/tdutils/test/ConcurrentHashMap.cpp | 252 ++++ .../Telegram/tdlib/td/tdutils/test/Enumerator.cpp | 2 +- .../tdutils/test/EpochBasedMemoryReclamation.cpp | 68 + .../Telegram/tdlib/td/tdutils/test/HashSet.cpp | 438 ++++++ .../tdlib/td/tdutils/test/HazardPointers.cpp | 16 +- .../Telegram/tdlib/td/tdutils/test/HttpUrl.cpp | 54 + protocols/Telegram/tdlib/td/tdutils/test/List.cpp | 169 +++ .../Telegram/tdlib/td/tdutils/test/MpmcQueue.cpp | 24 +- .../Telegram/tdlib/td/tdutils/test/MpmcWaiter.cpp | 60 +- .../tdlib/td/tdutils/test/MpscLinkQueue.cpp | 17 +- .../tdlib/td/tdutils/test/OptionParser.cpp | 82 ++ .../td/tdutils/test/OrderedEventsProcessor.cpp | 6 +- .../tdlib/td/tdutils/test/SharedObjectPool.cpp | 25 +- .../Telegram/tdlib/td/tdutils/test/SharedSlice.cpp | 91 ++ .../tdlib/td/tdutils/test/StealingQueue.cpp | 180 +++ .../tdlib/td/tdutils/test/WaitFreeHashMap.cpp | 95 ++ .../tdlib/td/tdutils/test/WaitFreeHashSet.cpp | 73 + .../tdlib/td/tdutils/test/WaitFreeVector.cpp | 69 + .../Telegram/tdlib/td/tdutils/test/bitmask.cpp | 249 ++++ .../Telegram/tdlib/td/tdutils/test/buffer.cpp | 56 + .../Telegram/tdlib/td/tdutils/test/crypto.cpp | 333 ++++- protocols/Telegram/tdlib/td/tdutils/test/emoji.cpp | 128 ++ .../Telegram/tdlib/td/tdutils/test/filesystem.cpp | 68 +- protocols/Telegram/tdlib/td/tdutils/test/gzip.cpp | 188 ++- .../tdlib/td/tdutils/test/hashset_benchmark.cpp | 647 +++++++++ protocols/Telegram/tdlib/td/tdutils/test/heap.cpp | 47 +- protocols/Telegram/tdlib/td/tdutils/test/json.cpp | 34 +- protocols/Telegram/tdlib/td/tdutils/test/log.cpp | 187 +++ protocols/Telegram/tdlib/td/tdutils/test/misc.cpp | 1218 ++++++++++++++-- protocols/Telegram/tdlib/td/tdutils/test/port.cpp | 316 +++++ protocols/Telegram/tdlib/td/tdutils/test/pq.cpp | 138 +- .../Telegram/tdlib/td/tdutils/test/variant.cpp | 24 +- 285 files changed, 27513 insertions(+), 5843 deletions(-) create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/AtomicRead.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/CancellationToken.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ChainScheduler.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/CombinedLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ConcurrentHashTable.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Context.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/DecTree.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Destructor.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/EpochBasedMemoryReclamation.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMap.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMapChunks.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashSet.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.h delete mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp delete mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Hash.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/HashMap.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/HashSet.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/HashTableUtils.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MapNode.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/NullLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.h delete mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/OptionsParser.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/PathView.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Promise.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/SetNode.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/SharedSlice.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/SharedSlice.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Slice.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/SliceBuilder.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Span.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/StealingQueue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ThreadLocalStorage.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ThreadSafeCounter.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TlDowncastHelper.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TlStorerToString.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsCerr.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsCerr.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsFileLog.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsFileLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsList.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsLog.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TsLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/UInt.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/VectorQueue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/WaitFreeHashMap.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/WaitFreeHashSet.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/WaitFreeVector.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/algorithm.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/as.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/bits.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/check.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/check.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/emoji.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/emoji.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/fixed_vector.h delete mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Fd.cpp delete mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Fd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/FromApp.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/IoSlice.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/MemoryMapping.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/MemoryMapping.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Mutex.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/PollFlags.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/PollFlags.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/StdStreams.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/StdStreams.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/UdpSocketFd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/UdpSocketFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Iocp.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Iocp.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/NativeFd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/NativeFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/PollableFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/ThreadPthread.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/skip_eintr.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/platform.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/rlimit.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/rlimit.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/stacktrace.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/stacktrace.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/uname.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/uname.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/user.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/user.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/tests.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/translit.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/translit.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/uint128.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/unique_ptr.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/ChainScheduler.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/ConcurrentHashMap.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/EpochBasedMemoryReclamation.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/HashSet.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/HttpUrl.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/List.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/OptionParser.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/SharedSlice.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/StealingQueue.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/WaitFreeHashMap.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/WaitFreeHashSet.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/WaitFreeVector.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/bitmask.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/buffer.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/emoji.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/hashset_benchmark.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/log.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/port.cpp (limited to 'protocols/Telegram/tdlib/td/tdutils') diff --git a/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt b/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt index 1fbc34df32..7c568fdeba 100644 --- a/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt +++ b/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt @@ -1,4 +1,12 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +if ((CMAKE_MAJOR_VERSION LESS 3) OR (CMAKE_VERSION VERSION_LESS "3.0.2")) + message(FATAL_ERROR "CMake >= 3.0.2 is required") +endif() + +option(TDUTILS_MIME_TYPE "Generate mime types conversion; requires gperf" ON) + +if (NOT DEFINED CMAKE_INSTALL_LIBDIR) + set(CMAKE_INSTALL_LIBDIR "lib") +endif() if (NOT ZLIB_FOUND) find_package(ZLIB) @@ -16,6 +24,17 @@ if (ZLIB_FOUND) endif() endif() +if (CRC32C_FOUND) + set(TD_HAVE_CRC32C 1) +endif() + +if (TD_WITH_ABSEIL) + find_package(ABSL QUIET) + if (ABSL_FOUND) + set(TD_HAVE_ABSL 1) + endif() +endif() + configure_file(td/utils/config.h.in td/utils/config.h @ONLY) add_subdirectory(generate) @@ -33,37 +52,55 @@ endif() set(TDUTILS_SOURCE td/utils/port/Clocks.cpp - td/utils/port/Fd.cpp td/utils/port/FileFd.cpp td/utils/port/IPAddress.cpp + td/utils/port/MemoryMapping.cpp td/utils/port/path.cpp + td/utils/port/platform.cpp + td/utils/port/PollFlags.cpp + td/utils/port/rlimit.cpp td/utils/port/ServerSocketFd.cpp td/utils/port/signals.cpp td/utils/port/sleep.cpp td/utils/port/SocketFd.cpp + td/utils/port/stacktrace.cpp td/utils/port/Stat.cpp + td/utils/port/StdStreams.cpp td/utils/port/thread_local.cpp + td/utils/port/UdpSocketFd.cpp + td/utils/port/uname.cpp + td/utils/port/user.cpp td/utils/port/wstring_convert.cpp td/utils/port/detail/Epoll.cpp td/utils/port/detail/EventFdBsd.cpp td/utils/port/detail/EventFdLinux.cpp td/utils/port/detail/EventFdWindows.cpp + td/utils/port/detail/Iocp.cpp td/utils/port/detail/KQueue.cpp + td/utils/port/detail/NativeFd.cpp td/utils/port/detail/Poll.cpp td/utils/port/detail/Select.cpp td/utils/port/detail/ThreadIdGuard.cpp + td/utils/port/detail/ThreadPthread.cpp td/utils/port/detail/WineventPoll.cpp ${TDMIME_AUTO} + td/utils/AsyncFileLog.cpp td/utils/base64.cpp td/utils/BigNum.cpp td/utils/buffer.cpp + td/utils/BufferedUdp.cpp + td/utils/check.cpp td/utils/crypto.cpp + td/utils/emoji.cpp + td/utils/ExitGuard.cpp td/utils/FileLog.cpp td/utils/filesystem.cpp td/utils/find_boundary.cpp + td/utils/FlatHashTable.cpp + td/utils/FloodControlGlobal.cpp td/utils/Gzip.cpp td/utils/GzipByteFlow.cpp td/utils/Hints.cpp @@ -71,14 +108,24 @@ set(TDUTILS_SOURCE td/utils/JsonBuilder.cpp td/utils/logging.cpp td/utils/misc.cpp - td/utils/MimeType.cpp + td/utils/MpmcQueue.cpp + td/utils/OptionParser.cpp + td/utils/PathView.cpp td/utils/Random.cpp + td/utils/SharedSlice.cpp + td/utils/Slice.cpp td/utils/StackAllocator.cpp td/utils/Status.cpp td/utils/StringBuilder.cpp + td/utils/tests.cpp td/utils/Time.cpp td/utils/Timer.cpp + td/utils/TsFileLog.cpp td/utils/tl_parsers.cpp + td/utils/translit.cpp + td/utils/TsCerr.cpp + td/utils/TsFileLog.cpp + td/utils/TsLog.cpp td/utils/unicode.cpp td/utils/utf8.cpp @@ -87,57 +134,98 @@ set(TDUTILS_SOURCE td/utils/port/CxCli.h td/utils/port/EventFd.h td/utils/port/EventFdBase.h - td/utils/port/Fd.h td/utils/port/FileFd.h + td/utils/port/FromApp.h td/utils/port/IPAddress.h + td/utils/port/IoSlice.h + td/utils/port/MemoryMapping.h + td/utils/port/Mutex.h td/utils/port/path.h td/utils/port/platform.h td/utils/port/Poll.h td/utils/port/PollBase.h + td/utils/port/PollFlags.h + td/utils/port/rlimit.h td/utils/port/RwMutex.h td/utils/port/ServerSocketFd.h td/utils/port/signals.h td/utils/port/sleep.h td/utils/port/SocketFd.h + td/utils/port/stacktrace.h td/utils/port/Stat.h + td/utils/port/StdStreams.h td/utils/port/thread.h td/utils/port/thread_local.h + td/utils/port/UdpSocketFd.h + td/utils/port/uname.h + td/utils/port/user.h td/utils/port/wstring_convert.h td/utils/port/detail/Epoll.h td/utils/port/detail/EventFdBsd.h td/utils/port/detail/EventFdLinux.h td/utils/port/detail/EventFdWindows.h + td/utils/port/detail/Iocp.h td/utils/port/detail/KQueue.h + td/utils/port/detail/NativeFd.h td/utils/port/detail/Poll.h + td/utils/port/detail/PollableFd.h td/utils/port/detail/Select.h + td/utils/port/detail/skip_eintr.h td/utils/port/detail/ThreadIdGuard.h td/utils/port/detail/ThreadPthread.h td/utils/port/detail/ThreadStl.h td/utils/port/detail/WineventPoll.h td/utils/AesCtrByteFlow.h + td/utils/algorithm.h + td/utils/as.h + td/utils/AsyncFileLog.h + td/utils/AtomicRead.h td/utils/base64.h td/utils/benchmark.h td/utils/BigNum.h + td/utils/bits.h td/utils/buffer.h td/utils/BufferedFd.h td/utils/BufferedReader.h + td/utils/BufferedUdp.h td/utils/ByteFlow.h + td/utils/CancellationToken.h + td/utils/ChainScheduler.h td/utils/ChangesProcessor.h + td/utils/check.h td/utils/Closure.h + td/utils/CombinedLog.h td/utils/common.h + td/utils/ConcurrentHashTable.h td/utils/Container.h + td/utils/Context.h td/utils/crypto.h + td/utils/DecTree.h + td/utils/Destructor.h + td/utils/emoji.h td/utils/Enumerator.h + td/utils/EpochBasedMemoryReclamation.h + td/utils/ExitGuard.h td/utils/FileLog.h td/utils/filesystem.h + td/utils/fixed_vector.h td/utils/find_boundary.h + td/utils/FlatHashMap.h + td/utils/FlatHashMapChunks.h + td/utils/FlatHashSet.h + td/utils/FlatHashTable.h td/utils/FloodControlFast.h + td/utils/FloodControlGlobal.h td/utils/FloodControlStrict.h td/utils/format.h td/utils/Gzip.h td/utils/GzipByteFlow.h + td/utils/Hash.h + td/utils/HashMap.h + td/utils/HashSet.h + td/utils/HashTableUtils.h td/utils/HazardPointers.h td/utils/Heap.h td/utils/Hints.h @@ -147,8 +235,8 @@ set(TDUTILS_SOURCE td/utils/JsonBuilder.h td/utils/List.h td/utils/logging.h + td/utils/MapNode.h td/utils/MemoryLog.h - td/utils/MimeType.h td/utils/misc.h td/utils/MovableValue.h td/utils/MpmcQueue.h @@ -156,68 +244,110 @@ set(TDUTILS_SOURCE td/utils/MpscPollableQueue.h td/utils/MpscLinkQueue.h td/utils/Named.h + td/utils/NullLog.h td/utils/ObjectPool.h td/utils/Observer.h td/utils/optional.h - td/utils/OptionsParser.h + td/utils/OptionParser.h td/utils/OrderedEventsProcessor.h td/utils/overloaded.h td/utils/Parser.h td/utils/PathView.h + td/utils/Promise.h td/utils/queue.h td/utils/Random.h td/utils/ScopeGuard.h + td/utils/SetNode.h td/utils/SharedObjectPool.h + td/utils/SharedSlice.h td/utils/Slice-decl.h td/utils/Slice.h + td/utils/SliceBuilder.h + td/utils/Span.h td/utils/SpinLock.h td/utils/StackAllocator.h td/utils/Status.h + td/utils/StealingQueue.h td/utils/Storer.h td/utils/StorerBase.h td/utils/StringBuilder.h td/utils/tests.h + td/utils/ThreadLocalStorage.h + td/utils/ThreadSafeCounter.h td/utils/Time.h td/utils/TimedStat.h td/utils/Timer.h td/utils/tl_helpers.h td/utils/tl_parsers.h td/utils/tl_storers.h + td/utils/TlDowncastHelper.h + td/utils/TlStorerToString.h + td/utils/translit.h + td/utils/TsCerr.h + td/utils/TsFileLog.h + td/utils/TsList.h + td/utils/TsLog.h td/utils/type_traits.h + td/utils/UInt.h + td/utils/uint128.h td/utils/unicode.h + td/utils/unique_ptr.h td/utils/utf8.h td/utils/Variant.h + td/utils/VectorQueue.h + td/utils/WaitFreeHashMap.h + td/utils/WaitFreeHashSet.h + td/utils/WaitFreeVector.h ) +if (TDUTILS_MIME_TYPE) + set(TDUTILS_SOURCE + ${TDUTILS_SOURCE} + td/utils/MimeType.cpp + td/utils/MimeType.h + ) +endif() + set(TDUTILS_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/bitmask.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/ChainScheduler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/emoji.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/EpochBasedMemoryReclamation.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/HashSet.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/HttpUrl.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/List.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/log.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/OptionParser.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedSlice.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/StealingQueue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/WaitFreeHashMap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/WaitFreeHashSet.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/WaitFreeVector.cpp PARENT_SCOPE ) -#RULES #LIBRARIES add_library(tdutils STATIC ${TDUTILS_SOURCE}) -if (WIN32) - # find_library(WS2_32_LIBRARY ws2_32) - # find_library(MSWSOCK_LIBRARY Mswsock) - # target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY}) - target_link_libraries(tdutils PRIVATE ws2_32 Mswsock) -endif() -if (NOT CMAKE_CROSSCOMPILING) + +if (NOT CMAKE_CROSSCOMPILING AND TDUTILS_MIME_TYPE) add_dependencies(tdutils tdmime_auto) endif() @@ -229,6 +359,14 @@ target_include_directories(tdutils PUBLIC $= 3.0.2 is required") +endif() # Generates files for MIME type <-> extension conversions -# DEPENDS ON: gperf grep bash/powershell +# DEPENDS ON: gperf grep + +if (NOT TDUTILS_MIME_TYPE) + return() +endif() file(MAKE_DIRECTORY auto) @@ -19,7 +25,7 @@ add_custom_target(tdmime_auto DEPENDS ${TDMIME_SOURCE}) if (NOT CMAKE_CROSSCOMPILING) find_program(GPERF_EXECUTABLE gperf) if (NOT GPERF_EXECUTABLE) - message(FATAL_ERROR "Could NOT find gperf. Path to gperf needs to be specified manually, i.e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"\" .')") + message(FATAL_ERROR "Could NOT find gperf. Add path to gperf executable to PATH environment variable or specify it manually using GPERF_EXECUTABLE option, i.e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"\"'.") endif() set(GPERF_FILES @@ -38,7 +44,7 @@ if (NOT CMAKE_CROSSCOMPILING) DEPENDS generate_mime_types_gperf mime_types.txt ) - if (WIN32) + if (CMAKE_HOST_WIN32) set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/mime_type_to_extension.cpp auto/mime_type_to_extension.gperf) else() set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 auto/mime_type_to_extension.gperf | grep -v __gnu_inline__ > auto/mime_type_to_extension.cpp) @@ -47,10 +53,10 @@ if (NOT CMAKE_CROSSCOMPILING) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ${MIME_TYPE_TO_EXTENSION_CMD} - DEPENDS auto/mime_type_to_extension.gperf + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.gperf ) - if (WIN32) + if (CMAKE_HOST_WIN32) set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/extension_to_mime_type.cpp auto/extension_to_mime_type.gperf) else() set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 auto/extension_to_mime_type.gperf | grep -v __gnu_inline__ > auto/extension_to_mime_type.cpp) @@ -59,6 +65,6 @@ if (NOT CMAKE_CROSSCOMPILING) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ${EXTENSION_TO_MIME_TYPE_CMD} - DEPENDS auto/extension_to_mime_type.gperf + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.gperf ) endif() diff --git a/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp b/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp index ac7ff68605..44ab2bd1e9 100644 --- a/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -15,7 +15,7 @@ #include #include -std::pair split(std::string s, char delimiter = ' ') { +static std::pair split(std::string s, char delimiter = ' ') { auto delimiter_pos = s.find(delimiter); if (delimiter_pos == std::string::npos) { return {std::move(s), ""}; @@ -26,8 +26,8 @@ std::pair split(std::string s, char delimiter = ' ') { } } -bool generate(const char *file_name, const char *from_name, const char *to_name, - const std::map &map) { +static bool generate(const char *file_name, const char *from_name, const char *to_name, + const std::map &map) { // binary mode is needed for MSYS2 gperf std::ofstream out(file_name, std::ios_base::trunc | std::ios_base::binary); if (!out) { @@ -71,6 +71,10 @@ bool generate(const char *file_name, const char *from_name, const char *to_name, return true; } +static bool is_private_mime_type(const std::string &mime_type) { + return mime_type.find("/x-") != std::string::npos; +} + int main(int argc, char *argv[]) { if (argc != 4) { std::cerr << "Wrong number of arguments supplied. Expected 'generate_mime_types_gperf " @@ -112,7 +116,7 @@ int main(int argc, char *argv[]) { std::vector extensions; while (!extensions_string.empty()) { - extensions.push_back(""); + extensions.emplace_back(); std::tie(extensions.back(), extensions_string) = split(extensions_string); } assert(!extensions.empty()); @@ -132,7 +136,13 @@ int main(int argc, char *argv[]) { for (auto &extension : extensions) { if (!extension_to_mime_type.emplace(extension, mime_type).second) { - std::cerr << "Extension \"" << extension << "\" matches more than one type" << std::endl; + if (is_private_mime_type(extension_to_mime_type[extension]) == is_private_mime_type(mime_type)) { + std::cerr << "Extension \"" << extension << "\" matches more than one type" << std::endl; + } else { + if (!is_private_mime_type(mime_type)) { + extension_to_mime_type[extension] = mime_type; + } + } } } } diff --git a/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt b/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt index a2d4abbf29..18dcfb775b 100644 --- a/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt +++ b/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt @@ -14,7 +14,7 @@ application/davmount+xml davmount application/docbook+xml dbk application/dssc+der dssc application/dssc+xml xdssc -application/ecmascript ecma +application/ecmascript es application/emma+xml emma application/epub+zip epub application/exi exi @@ -498,6 +498,7 @@ application/x-dtbresource+xml res application/x-dvi dvi application/x-envoy evy application/x-eva eva +application/x-fictionbook+xml fb2 application/x-font-bdf bdf application/x-font-ghostscript gsf application/x-font-linux-psf psf @@ -506,7 +507,7 @@ application/x-font-pcf pcf application/x-font-snf snf application/x-font-ttf ttf ttc application/x-font-type1 pfa pfb pfm afm -application/font-woff woff +application/x-font-woff woff application/x-freearc arc application/x-futuresplash spl application/x-gca-compressed gca @@ -564,6 +565,8 @@ application/x-tex tex application/x-tex-tfm tfm application/x-texinfo texinfo texi application/x-tgif obj +application/x-tgsticker tgs +application/x-tgwallpattern tgv application/x-ustar ustar application/x-wais-source src application/x-x509-ca-cert der crt @@ -589,9 +592,9 @@ application/zip zip audio/adpcm adp audio/basic au snd audio/midi mid midi kar rmi -audio/mp4 mp4a +audio/mp4 m4a mp4a audio/mpeg mpga mp2 mp2a mp3 m2a m3a -audio/ogg oga ogg spx +audio/ogg oga ogg opus spx audio/s3m s3m audio/silk sil audio/vnd.dece.audio uva uvva @@ -624,10 +627,19 @@ chemical/x-cmdf cmdf chemical/x-cml cml chemical/x-csml csml chemical/x-xyz xyz +font/collection ttc +font/otf otf +font/ttf ttf +font/woff woff +font/woff2 woff2 image/bmp bmp image/cgm cgm image/g3fax g3 image/gif gif +image/heic heic +image/heic-sequence heics +image/heif heif +image/heif-sequence heifs image/ief ief image/jpeg jpeg jpg jpe image/ktx ktx @@ -638,8 +650,8 @@ image/svg+xml svg svgz image/tiff tiff tif image/vnd.adobe.photoshop psd image/vnd.dece.graphic uvi uvvi uvg uvvg -image/vnd.dvb.subtitle sub image/vnd.djvu djvu djv +image/vnd.dvb.subtitle sub image/vnd.dwg dwg image/vnd.dxf dxf image/vnd.fastbidsheet fbs @@ -700,8 +712,8 @@ text/uri-list uri uris urls text/vcard vcard text/vnd.curl curl text/vnd.curl.dcurl dcurl -text/vnd.curl.scurl scurl text/vnd.curl.mcurl mcurl +text/vnd.curl.scurl scurl text/vnd.dvb.subtitle sub text/vnd.fly fly text/vnd.fmi.flexstor flx @@ -715,9 +727,10 @@ text/x-asm s asm text/x-c c cc cxx cpp h hh dic text/x-fortran f for f77 f90 text/x-java-source java +text/x-nfo nfo text/x-opml opml text/x-pascal p pas -text/x-nfo nfo +text/x-php php text/x-setext etx text/x-sfv sfv text/x-uuencode uu @@ -728,6 +741,7 @@ video/3gpp2 3g2 video/h261 h261 video/h263 h263 video/h264 h264 +video/h265 h265 video/jpeg jpgv video/jpm jpm jpgm video/mj2 mj2 mjp2 diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h index 820aa02fbe..5d9057f215 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -11,14 +11,15 @@ #include "td/utils/crypto.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" +#include "td/utils/UInt.h" namespace td { #if TD_HAVE_OPENSSL -class AesCtrByteFlow : public ByteFlowInplaceBase { +class AesCtrByteFlow final : public ByteFlowInplaceBase { public: void init(const UInt256 &key, const UInt128 &iv) { - state_.init(key, iv); + state_.init(as_slice(key), as_slice(iv)); } void init(AesCtrState &&state) { state_ = std::move(state); @@ -26,25 +27,20 @@ class AesCtrByteFlow : public ByteFlowInplaceBase { AesCtrState move_aes_ctr_state() { return std::move(state_); } - void loop() override { - bool was_updated = false; - while (true) { - auto ready = input_->prepare_read(); - if (ready.empty()) { - break; - } + bool loop() final { + bool result = false; + auto ready = input_->prepare_read(); + if (!ready.empty()) { state_.encrypt(ready, MutableSlice(const_cast(ready.data()), ready.size())); input_->confirm_read(ready.size()); output_.advance_end(ready.size()); - was_updated = true; - } - if (was_updated) { - on_output_updated(); + result = true; } + if (!is_input_active_) { finish(Status::OK()); // End of input stream. } - set_need_size(1); + return result; } private: diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.cpp new file mode 100644 index 0000000000..575333745d --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.cpp @@ -0,0 +1,160 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/AsyncFileLog.h" + +#include "td/utils/port/FileFd.h" +#include "td/utils/port/path.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/SliceBuilder.h" +#include "td/utils/Time.h" + +namespace td { + +#if !TD_THREAD_UNSUPPORTED + +Status AsyncFileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) { + CHECK(path_.empty()); + CHECK(!path.empty()); + + TRY_RESULT(fd, FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append)); + if (!Stderr().empty() && redirect_stderr) { + fd.get_native_fd().duplicate(Stderr().get_native_fd()).ignore(); + } + + auto r_path = realpath(path, true); + if (r_path.is_error()) { + path_ = std::move(path); + } else { + path_ = r_path.move_as_ok(); + } + TRY_RESULT(size, fd.get_size()); + + queue_ = td::make_unique>(); + queue_->init(); + + logging_thread_ = td::thread( + [queue = queue_.get(), fd = std::move(fd), path = path_, size, rotate_threshold, redirect_stderr]() mutable { + auto after_rotation = [&] { + fd.close(); + auto r_fd = FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append); + if (r_fd.is_error()) { + process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__ << '\n'); + } + fd = r_fd.move_as_ok(); + if (!Stderr().empty() && redirect_stderr) { + fd.get_native_fd().duplicate(Stderr().get_native_fd()).ignore(); + } + size = 0; + }; + auto append = [&](CSlice slice) { + if (size > rotate_threshold) { + auto status = rename(path, PSLICE() << path << ".old"); + if (status.is_error()) { + process_fatal_error(PSLICE() << status << " in " << __FILE__ << " at " << __LINE__ << '\n'); + } + after_rotation(); + } + while (!slice.empty()) { + if (redirect_stderr) { + while (has_log_guard()) { + // spin + } + } + auto r_size = fd.write(slice); + if (r_size.is_error()) { + process_fatal_error(PSLICE() << r_size.error() << " in " << __FILE__ << " at " << __LINE__ << '\n'); + } + auto written = r_size.ok(); + size += static_cast(written); + slice.remove_prefix(written); + } + }; + + while (true) { + int ready_count = queue->reader_wait_nonblock(); + if (ready_count == 0) { + queue->reader_get_event_fd().wait(1000); + continue; + } + bool need_close = false; + while (ready_count-- > 0) { + Query query = queue->reader_get_unsafe(); + switch (query.type_) { + case Query::Type::Log: + append(query.data_); + break; + case Query::Type::AfterRotation: + after_rotation(); + break; + case Query::Type::Close: + need_close = true; + break; + default: + process_fatal_error("Invalid query type in AsyncFileLog"); + } + } + queue->reader_flush(); + + if (need_close) { + fd.close(); + break; + } + } + }); + + return Status::OK(); +} + +AsyncFileLog::~AsyncFileLog() { + if (queue_ == nullptr) { + return; + } + Query query; + query.type_ = Query::Type::Close; + queue_->writer_put(std::move(query)); + logging_thread_.join(); +} + +vector AsyncFileLog::get_file_paths() { + vector result; + if (!path_.empty()) { + result.push_back(path_); + result.push_back(PSTRING() << path_ << ".old"); + } + return result; +} + +void AsyncFileLog::after_rotation() { + Query query; + query.type_ = Query::Type::AfterRotation; + if (queue_ == nullptr) { + process_fatal_error("AsyncFileLog is not inited"); + } + queue_->writer_put(std::move(query)); +} + +void AsyncFileLog::do_append(int log_level, CSlice slice) { + Query query; + query.data_ = slice.str(); + if (queue_ == nullptr) { + process_fatal_error("AsyncFileLog is not inited"); + } + queue_->writer_put(std::move(query)); + if (log_level == VERBOSITY_NAME(FATAL)) { + // it is not thread-safe to join logging_thread_ there, so just wait for the log line to be printed + auto end_time = Time::now() + 1.0; + while (!queue_->is_empty() && Time::now() < end_time) { + usleep_for(1000); + } + usleep_for(5000); // allow some time for the log line to be actually printed + } +} + +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.h new file mode 100644 index 0000000000..1b0ab73897 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/AsyncFileLog.h @@ -0,0 +1,51 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/MpscPollableQueue.h" +#include "td/utils/port/thread.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +#if !TD_THREAD_UNSUPPORTED + +class AsyncFileLog final : public LogInterface { + public: + AsyncFileLog() = default; + AsyncFileLog(const AsyncFileLog &) = delete; + AsyncFileLog &operator=(const AsyncFileLog &) = delete; + AsyncFileLog(AsyncFileLog &&) = delete; + AsyncFileLog &operator=(AsyncFileLog &&) = delete; + ~AsyncFileLog(); + + Status init(string path, int64 rotate_threshold, bool redirect_stderr = true); + + private: + struct Query { + enum class Type : int32 { Log, AfterRotation, Close }; + Type type_ = Type::Log; + string data_; + }; + + string path_; + unique_ptr> queue_; + thread logging_thread_; + + vector get_file_paths() final; + + void after_rotation() final; + + void do_append(int log_level, CSlice slice) final; +}; + +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/AtomicRead.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/AtomicRead.h new file mode 100644 index 0000000000..d30e960a8c --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/AtomicRead.h @@ -0,0 +1,89 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/sleep.h" +#include "td/utils/type_traits.h" + +#include +#include +#include + +namespace td { + +template +class AtomicRead { + public: + void read(T &dest) const { + uint32 counter = 0; + auto wait = [&] { + counter++; + const int wait_each_count = 4; + if (counter % wait_each_count == 0) { + usleep_for(1); + } + }; + + while (true) { + static_assert(TD_IS_TRIVIALLY_COPYABLE(T), "T must be trivially copyable"); + auto version_before = version.load(); + if (version_before % 2 == 0) { + std::memcpy(&dest, &value, sizeof(dest)); + auto version_after = version.load(); + if (version_before == version_after) { + break; + } + } + wait(); + } + } + + struct Write { + explicit Write(AtomicRead *read) { + read->do_lock(); + ptr.reset(read); + } + struct Destructor { + void operator()(AtomicRead *read) const { + read->do_unlock(); + } + }; + T &operator*() { + return value(); + } + T *operator->() { + return &value(); + } + T &value() { + CHECK(ptr); + return ptr->value; + } + + private: + std::unique_ptr ptr; + }; + + Write lock() { + return Write(this); + } + + private: + std::atomic version{0}; + T value; + + void do_lock() { + bool is_locked = ++version % 2 == 1; + CHECK(is_locked); + } + void do_unlock() { + bool is_unlocked = ++version % 2 == 0; + CHECK(is_unlocked); + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp index f553661d49..e7a93d4399 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -12,10 +12,13 @@ char disable_linker_warning_about_empty_file_bignum_cpp TD_UNUSED; #include "td/utils/logging.h" #include "td/utils/misc.h" +#include "td/utils/SliceBuilder.h" #include #include +#include + namespace td { class BigNumContext::Impl { @@ -37,9 +40,8 @@ class BigNumContext::Impl { BigNumContext::BigNumContext() : impl_(make_unique()) { } -BigNumContext::BigNumContext(BigNumContext &&other) = default; -BigNumContext &BigNumContext::operator=(BigNumContext &&other) = default; - +BigNumContext::BigNumContext(BigNumContext &&other) noexcept = default; +BigNumContext &BigNumContext::operator=(BigNumContext &&other) noexcept = default; BigNumContext::~BigNumContext() = default; class BigNum::Impl { @@ -68,6 +70,9 @@ BigNum::BigNum(const BigNum &other) : BigNum() { } BigNum &BigNum::operator=(const BigNum &other) { + if (this == &other) { + return *this; + } CHECK(impl_ != nullptr); CHECK(other.impl_ != nullptr); BIGNUM *result = BN_copy(impl_->big_num, other.impl_->big_num); @@ -75,20 +80,41 @@ BigNum &BigNum::operator=(const BigNum &other) { return *this; } -BigNum::BigNum(BigNum &&other) = default; - -BigNum &BigNum::operator=(BigNum &&other) = default; - +BigNum::BigNum(BigNum &&other) noexcept = default; +BigNum &BigNum::operator=(BigNum &&other) noexcept = default; BigNum::~BigNum() = default; BigNum BigNum::from_binary(Slice str) { return BigNum(make_unique(BN_bin2bn(str.ubegin(), narrow_cast(str.size()), nullptr))); } -BigNum BigNum::from_decimal(CSlice str) { +BigNum BigNum::from_le_binary(Slice str) { +#if defined(OPENSSL_IS_BORINGSSL) + return BigNum(make_unique(BN_le2bn(str.ubegin(), narrow_cast(str.size()), nullptr))); +#elif OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) + return BigNum(make_unique(BN_lebin2bn(str.ubegin(), narrow_cast(str.size()), nullptr))); +#else + string str_copy = str.str(); + std::reverse(str_copy.begin(), str_copy.end()); + return from_binary(str_copy); +#endif +} + +Result BigNum::from_decimal(CSlice str) { BigNum result; - int err = BN_dec2bn(&result.impl_->big_num, str.c_str()); - LOG_IF(FATAL, err == 0); + int res = BN_dec2bn(&result.impl_->big_num, str.c_str()); + if (res == 0 || static_cast(res) != str.size()) { + return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as BigNum"); + } + return result; +} + +Result BigNum::from_hex(CSlice str) { + BigNum result; + int res = BN_hex2bn(&result.impl_->big_num, str.c_str()); + if (res == 0 || static_cast(res) != str.size()) { + return Status::Error(PSLICE() << "Failed to parse \"" << str << "\" as hexadecimal BigNum"); + } return result; } @@ -99,10 +125,6 @@ BigNum BigNum::from_raw(void *openssl_big_num) { BigNum::BigNum(unique_ptr &&impl) : impl_(std::move(impl)) { } -void BigNum::ensure_const_time() { - BN_set_flags(impl_->big_num, BN_FLG_CONSTTIME); -} - int BigNum::get_num_bits() const { return BN_num_bits(impl_->big_num); } @@ -126,7 +148,12 @@ bool BigNum::is_bit_set(int num) const { } bool BigNum::is_prime(BigNumContext &context) const { - int result = BN_is_prime_ex(impl_->big_num, BN_prime_checks, context.impl_->big_num_context, nullptr); +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER) + int result = BN_check_prime(impl_->big_num, context.impl_->big_num_context, nullptr); +#else + int result = + BN_is_prime_ex(impl_->big_num, get_num_bits() > 2048 ? 128 : 64, context.impl_->big_num_context, nullptr); +#endif LOG_IF(FATAL, result == -1); return result == 1; } @@ -180,10 +207,32 @@ string BigNum::to_binary(int exact_size) const { CHECK(exact_size >= num_size); } string res(exact_size, '\0'); - BN_bn2bin(impl_->big_num, reinterpret_cast(&res[exact_size - num_size])); + BN_bn2bin(impl_->big_num, MutableSlice(res).ubegin() + (exact_size - num_size)); return res; } +string BigNum::to_le_binary(int exact_size) const { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) || defined(OPENSSL_IS_BORINGSSL) + int num_size = get_num_bytes(); + if (exact_size == -1) { + exact_size = num_size; + } else { + CHECK(exact_size >= num_size); + } + string result(exact_size, '\0'); +#if defined(OPENSSL_IS_BORINGSSL) + BN_bn2le_padded(MutableSlice(result).ubegin(), exact_size, impl_->big_num); +#else + BN_bn2lebinpad(impl_->big_num, MutableSlice(result).ubegin(), exact_size); +#endif + return result; +#else + string result = to_binary(exact_size); + std::reverse(result.begin(), result.end()); + return result; +#endif +} + string BigNum::to_decimal() const { char *result = BN_bn2dec(impl_->big_num); CHECK(result != nullptr); @@ -214,12 +263,29 @@ void BigNum::mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) { LOG_IF(FATAL, result != 1); } +void BigNum::mod_add(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) { + int result = BN_mod_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num, + context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mod_sub(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) { + int result = BN_mod_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num, + context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) { int result = BN_mod_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context); LOG_IF(FATAL, result != 1); } +void BigNum::mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context) { + auto result = BN_mod_inverse(r.impl_->big_num, a.impl_->big_num, m.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != r.impl_->big_num); +} + void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, BigNumContext &context) { auto q = quotient == nullptr ? nullptr : quotient->impl_->big_num; @@ -247,5 +313,9 @@ int BigNum::compare(const BigNum &a, const BigNum &b) { return BN_cmp(a.impl_->big_num, b.impl_->big_num); } +StringBuilder &operator<<(StringBuilder &sb, const BigNum &bn) { + return sb << bn.to_decimal(); +} + } // namespace td #endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h index 6eecdeab03..9b666f4ce0 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -11,6 +11,8 @@ #if TD_HAVE_OPENSSL #include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" namespace td { @@ -19,8 +21,8 @@ class BigNumContext { BigNumContext(); BigNumContext(const BigNumContext &other) = delete; BigNumContext &operator=(const BigNumContext &other) = delete; - BigNumContext(BigNumContext &&other); - BigNumContext &operator=(BigNumContext &&other); + BigNumContext(BigNumContext &&other) noexcept; + BigNumContext &operator=(BigNumContext &&other) noexcept; ~BigNumContext(); private: @@ -35,20 +37,22 @@ class BigNum { BigNum(); BigNum(const BigNum &other); BigNum &operator=(const BigNum &other); - BigNum(BigNum &&other); - BigNum &operator=(BigNum &&other); + BigNum(BigNum &&other) noexcept; + BigNum &operator=(BigNum &&other) noexcept; ~BigNum(); static BigNum from_binary(Slice str); - static BigNum from_decimal(CSlice str); + static BigNum from_le_binary(Slice str); + + static Result from_decimal(CSlice str); + + static Result from_hex(CSlice str); static BigNum from_raw(void *openssl_big_num); void set_value(uint32 new_value); - void ensure_const_time(); - int get_num_bits() const; int get_num_bytes() const; @@ -65,6 +69,8 @@ class BigNum { string to_binary(int exact_size = -1) const; + string to_le_binary(int exact_size = -1) const; + string to_decimal() const; void operator+=(uint32 value); @@ -85,8 +91,14 @@ class BigNum { static void mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context); + static void mod_add(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); + + static void mod_sub(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); + static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); + static void mod_inverse(BigNum &r, BigNum &a, const BigNum &m, BigNumContext &context); + static void div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, BigNumContext &context); @@ -103,6 +115,8 @@ class BigNum { explicit BigNum(unique_ptr &&impl); }; +StringBuilder &operator<<(StringBuilder &sb, const BigNum &bn); + } // namespace td #endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h index 0c8f65408d..6bb9b77098 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,10 +7,13 @@ #pragma once #include "td/utils/buffer.h" +#include "td/utils/common.h" #include "td/utils/format.h" #include "td/utils/logging.h" -#include "td/utils/port/Fd.h" +#include "td/utils/port/detail/PollableFd.h" +#include "td/utils/port/IoSlice.h" #include "td/utils/Slice.h" +#include "td/utils/Span.h" #include "td/utils/Status.h" #include @@ -28,15 +31,16 @@ class BufferedFdBase : public FdT { Result flush_write() TD_WARN_UNUSED_RESULT; bool need_flush_write(size_t at_least = 0) { - CHECK(write_); - write_->sync_with_writer(); - return write_->size() > at_least; + return ready_for_flush_write() > at_least; } size_t ready_for_flush_write() { CHECK(write_); write_->sync_with_writer(); return write_->size(); } + void sync_with_poll() { + ::td::sync_with_poll(*this); + } void set_input_writer(ChainBufferWriter *read) { read_ = read; } @@ -50,7 +54,7 @@ class BufferedFdBase : public FdT { }; template -class BufferedFd : public BufferedFdBase { +class BufferedFd final : public BufferedFdBase { using Parent = BufferedFdBase; ChainBufferWriter input_writer_; ChainBufferReader input_reader_; @@ -62,14 +66,21 @@ class BufferedFd : public BufferedFdBase { public: BufferedFd(); explicit BufferedFd(FdT &&fd_); - BufferedFd(BufferedFd &&); - BufferedFd &operator=(BufferedFd &&); + BufferedFd(BufferedFd &&) noexcept; + BufferedFd &operator=(BufferedFd &&) noexcept; BufferedFd(const BufferedFd &) = delete; BufferedFd &operator=(const BufferedFd &) = delete; ~BufferedFd(); void close(); + size_t left_unread() const { + return input_reader_.size(); + } + size_t left_unwritten() const { + return output_reader_.size(); + } + Result flush_read(size_t max_read = std::numeric_limits::max()) TD_WARN_UNUSED_RESULT; Result flush_write() TD_WARN_UNUSED_RESULT; @@ -89,8 +100,9 @@ template Result BufferedFdBase::flush_read(size_t max_read) { CHECK(read_); size_t result = 0; - while (::td::can_read(*this) && max_read) { - MutableSlice slice = read_->prepare_append().truncate(max_read); + while (::td::can_read_local(*this) && max_read) { + MutableSlice slice = read_->prepare_append(); + slice.truncate(max_read); TRY_RESULT(x, FdT::read(slice)); slice.truncate(x); read_->confirm_append(x); @@ -102,13 +114,25 @@ Result BufferedFdBase::flush_read(size_t max_read) { template Result BufferedFdBase::flush_write() { - size_t result = 0; // TODO: sync on demand write_->sync_with_writer(); - while (!write_->empty() && ::td::can_write(*this)) { - Slice slice = write_->prepare_read(); - TRY_RESULT(x, FdT::write(slice)); - write_->confirm_read(x); + size_t result = 0; + while (!write_->empty() && ::td::can_write_local(*this)) { + constexpr size_t BUF_SIZE = 20; + IoSlice buf[BUF_SIZE]; + + auto it = write_->clone(); + size_t buf_i; + for (buf_i = 0; buf_i < BUF_SIZE; buf_i++) { + Slice slice = it.prepare_read(); + if (slice.empty()) { + break; + } + buf[buf_i] = as_io_slice(slice); + it.confirm_read(slice.size()); + } + TRY_RESULT(x, FdT::writev(Span(buf, buf_i))); + write_->advance(x); result += x; } return result; @@ -139,12 +163,12 @@ BufferedFd::BufferedFd(FdT &&fd_) : Parent(std::move(fd_)) { } template -BufferedFd::BufferedFd(BufferedFd &&from) { +BufferedFd::BufferedFd(BufferedFd &&from) noexcept { *this = std::move(from); } template -BufferedFd &BufferedFd::operator=(BufferedFd &&from) { +BufferedFd &BufferedFd::operator=(BufferedFd &&from) noexcept { FdT::operator=(std::move(static_cast(from))); input_reader_ = std::move(from.input_reader_); input_writer_ = std::move(from.input_writer_); @@ -171,7 +195,7 @@ Result BufferedFd::flush_read(size_t max_read) { if (result) { // TODO: faster sync is possible if you owns writer. input_reader_.sync_with_writer(); - LOG(DEBUG) << "flush_read: +" << format::as_size(result) << tag("total", format::as_size(input_reader_.size())); + LOG(DEBUG) << "Flush read: +" << format::as_size(result) << tag("total", format::as_size(input_reader_.size())); } return result; } @@ -180,7 +204,7 @@ template Result BufferedFd::flush_write() { TRY_RESULT(result, Parent::flush_write()); if (result) { - LOG(DEBUG) << "flush_write: +" << format::as_size(result) << tag("left", format::as_size(output_reader_.size())); + LOG(DEBUG) << "Flush write: +" << format::as_size(result) << tag("left", format::as_size(output_reader_.size())); } return result; } diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h index 9006d78132..5fd8ac44fe 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -11,12 +11,11 @@ #include "td/utils/Slice.h" #include "td/utils/Status.h" -#include - namespace td { + class BufferedReader { public: - explciit BufferedReader(FileFd &file, size_t buff_size = 8152) + explicit BufferedReader(FileFd &file, size_t buff_size = 8152) : file_(file), buff_(buff_size), begin_pos_(0), end_pos_(0) { } @@ -33,13 +32,13 @@ inline Result BufferedReader::read(MutableSlice slice) { size_t available = end_pos_ - begin_pos_; if (available >= slice.size()) { // have enough data in buffer - std::memcpy(slice.begin(), &buff_[begin_pos_], slice.size()); + slice.copy_from({&buff_[begin_pos_], slice.size()}); begin_pos_ += slice.size(); return slice.size(); } if (available) { - std::memcpy(slice.begin(), &buff_[begin_pos_], available); + slice.copy_from({&buff_[begin_pos_], available}); begin_pos_ += available; slice.remove_prefix(available); } @@ -54,8 +53,9 @@ inline Result BufferedReader::read(MutableSlice slice) { end_pos_ = result; size_t left = min(end_pos_, slice.size()); - std::memcpy(slice.begin(), &buff_[begin_pos_], left); - begin_pos_ += left; + slice.copy_from({&buff_[0], left}); + begin_pos_ = left; return left + available; } + } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.cpp new file mode 100644 index 0000000000..e28bb97069 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.cpp @@ -0,0 +1,17 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/BufferedUdp.h" + +char disable_linker_warning_about_empty_file_buffered_udp_cpp TD_UNUSED; + +namespace td { + +#if TD_PORT_POSIX +TD_THREAD_LOCAL detail::UdpReader *BufferedUdp::udp_reader_; +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.h new file mode 100644 index 0000000000..5907e17557 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedUdp.h @@ -0,0 +1,177 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/optional.h" +#include "td/utils/port/detail/PollableFd.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/port/UdpSocketFd.h" +#include "td/utils/Span.h" +#include "td/utils/Status.h" +#include "td/utils/VectorQueue.h" + +#include + +namespace td { + +#if TD_PORT_POSIX +namespace detail { +class UdpWriter { + public: + static Status write_once(UdpSocketFd &fd, VectorQueue &queue) TD_WARN_UNUSED_RESULT { + std::array messages; + auto to_send = queue.as_span(); + size_t to_send_n = td::min(messages.size(), to_send.size()); + to_send.truncate(to_send_n); + for (size_t i = 0; i < to_send_n; i++) { + messages[i].to = &to_send[i].address; + messages[i].data = to_send[i].data.as_slice(); + } + + size_t cnt; + auto status = fd.send_messages(Span(messages).truncate(to_send_n), cnt); + queue.pop_n(cnt); + return status; + } +}; + +class UdpReaderHelper { + public: + void init_inbound_message(UdpSocketFd::InboundMessage &message) { + message.from = &message_.address; + message.error = &message_.error; + if (buffer_.size() < MAX_PACKET_SIZE) { + buffer_ = BufferSlice(RESERVED_SIZE); + } + CHECK(buffer_.size() >= MAX_PACKET_SIZE); + message.data = buffer_.as_slice().substr(0, MAX_PACKET_SIZE); + } + + UdpMessage extract_udp_message(UdpSocketFd::InboundMessage &message) { + message_.data = buffer_.from_slice(message.data); + auto size = message_.data.size(); + size = (size + 7) & ~7; + CHECK(size <= MAX_PACKET_SIZE); + buffer_.confirm_read(size); + return std::move(message_); + } + + private: + static constexpr size_t MAX_PACKET_SIZE = 2048; + static constexpr size_t RESERVED_SIZE = MAX_PACKET_SIZE * 8; + UdpMessage message_; + BufferSlice buffer_; +}; + +// One for thread is enough +class UdpReader { + public: + UdpReader() { + for (size_t i = 0; i < messages_.size(); i++) { + helpers_[i].init_inbound_message(messages_[i]); + } + } + Status read_once(UdpSocketFd &fd, VectorQueue &queue) TD_WARN_UNUSED_RESULT { + for (auto &message : messages_) { + CHECK(message.data.size() == 2048); + } + size_t cnt = 0; + auto status = fd.receive_messages(messages_, cnt); + for (size_t i = 0; i < cnt; i++) { + queue.push(helpers_[i].extract_udp_message(messages_[i])); + helpers_[i].init_inbound_message(messages_[i]); + } + for (size_t i = cnt; i < messages_.size(); i++) { + LOG_CHECK(messages_[i].data.size() == 2048) + << " cnt = " << cnt << " i = " << i << " size = " << messages_[i].data.size() << " status = " << status; + } + if (status.is_error() && !UdpSocketFd::is_critical_read_error(status)) { + queue.push(UdpMessage{{}, {}, std::move(status)}); + status = Status::OK(); + } + return status; + } + + private: + static constexpr size_t BUFFER_SIZE = 16; + std::array messages_; + std::array helpers_; +}; + +} // namespace detail + +#endif + +class BufferedUdp final : public UdpSocketFd { + public: + explicit BufferedUdp(UdpSocketFd fd) : UdpSocketFd(std::move(fd)) { + } + +#if TD_PORT_POSIX + void sync_with_poll() { + ::td::sync_with_poll(*this); + } + Result> receive() { + if (input_.empty() && can_read_local(*this)) { + TRY_STATUS(flush_read_once()); + } + if (input_.empty()) { + return optional(); + } + return input_.pop(); + } + + void send(UdpMessage message) { + output_.push(std::move(message)); + } + + Status flush_send() { + Status status; + while (status.is_ok() && can_write_local(*this) && !output_.empty()) { + status = flush_send_once(); + } + return status; + } +#endif + + UdpSocketFd move_as_udp_socket_fd() { + return std::move(as_fd()); + } + + UdpSocketFd &as_fd() { + return *static_cast(this); + } + + private: +#if TD_PORT_POSIX + VectorQueue input_; + VectorQueue output_; + + VectorQueue &input() { + return input_; + } + VectorQueue &output() { + return output_; + } + + Status flush_send_once() TD_WARN_UNUSED_RESULT { + return detail::UdpWriter::write_once(as_fd(), output_); + } + + Status flush_read_once() TD_WARN_UNUSED_RESULT { + init_thread_local(udp_reader_); + return udp_reader_->read_once(as_fd(), input_); + } + + static TD_THREAD_LOCAL detail::UdpReader *udp_reader_; +#endif +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h index fb0c4489eb..2043864656 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -8,9 +8,10 @@ #include "td/utils/buffer.h" #include "td/utils/common.h" -#include "td/utils/logging.h" #include "td/utils/Status.h" +#include + namespace td { class ByteFlowInterface { @@ -20,6 +21,10 @@ class ByteFlowInterface { virtual void set_parent(ByteFlowInterface &other) = 0; virtual void set_input(ChainBufferReader *input) = 0; virtual size_t get_need_size() = 0; + virtual size_t get_read_size() = 0; + virtual size_t get_write_size() = 0; + virtual void reset_need_size() { + } ByteFlowInterface() = default; ByteFlowInterface(const ByteFlowInterface &) = delete; ByteFlowInterface &operator=(const ByteFlowInterface &) = delete; @@ -42,36 +47,98 @@ class ByteFlowBaseCommon : public ByteFlowInterface { } void wakeup() final { - if (stop_flag_) { + if (stop_flag_ || !input_) { return; } input_->sync_with_writer(); + if (waiting_flag_) { if (!is_input_active_) { finish(Status::OK()); } return; } - if (is_input_active_) { - if (need_size_ != 0 && input_->size() < need_size_) { - return; + while (true) { + if (stop_flag_) { + break; + } + + // update can_read + if (is_input_active_) { + auto read_size = get_read_size(); + if (read_size < min(need_size_, options_.read_watermark.low)) { + can_read = false; + } + if (read_size >= max(need_size_, options_.read_watermark.high)) { + can_read = true; + } + } else { + // always can read when input is closed + can_read = true; + } + + // update can_write + { + auto write_size = get_write_size(); + if (write_size > options_.write_watermark.high) { + can_write = false; + } + if (write_size <= options_.write_watermark.low) { + can_write = true; + } + } + + if (!can_read || !can_write) { + break; + } + need_size_ = 0; + + if (!loop()) { + if (need_size_ <= get_read_size()) { + need_size_ = get_read_size() + 1; + } } } - need_size_ = 0; - loop(); + on_output_updated(); } size_t get_need_size() final { return need_size_; } + void reset_need_size() override { + need_size_ = 0; + } + size_t get_read_size() override { + input_->sync_with_writer(); + return input_->size(); + } + size_t get_write_size() override { + CHECK(parent_); + return parent_->get_read_size(); + } + + struct Watermark { + size_t low{std::numeric_limits::max()}; + size_t high{0}; + }; + struct Options { + Watermark write_watermark; + Watermark read_watermark; + }; + void set_options(Options options) { + options_ = options; + } - virtual void loop() = 0; + virtual bool loop() = 0; protected: bool waiting_flag_ = false; - ChainBufferReader *input_; + ChainBufferReader *input_ = nullptr; bool is_input_active_ = true; size_t need_size_ = 0; + bool can_read{true}; + bool can_write{true}; + Options options_; void finish(Status status) { stop_flag_ = true; need_size_ = 0; @@ -115,7 +182,7 @@ class ByteFlowBase : public ByteFlowBaseCommon { parent_ = &other; parent_->set_input(&output_reader_); } - void loop() override = 0; + bool loop() override = 0; // ChainBufferWriter &get_output() { // return output_; @@ -138,7 +205,7 @@ class ByteFlowInplaceBase : public ByteFlowBaseCommon { parent_ = &other; parent_->set_input(&output_); } - void loop() override = 0; + bool loop() override = 0; ChainBufferReader &get_output() { return output_; @@ -153,16 +220,16 @@ inline ByteFlowInterface &operator>>(ByteFlowInterface &from, ByteFlowInterface return to; } -class ByteFlowSource : public ByteFlowInterface { +class ByteFlowSource final : public ByteFlowInterface { public: ByteFlowSource() = default; explicit ByteFlowSource(ChainBufferReader *buffer) : buffer_(buffer) { } - ByteFlowSource(ByteFlowSource &&other) : buffer_(other.buffer_), parent_(other.parent_) { + ByteFlowSource(ByteFlowSource &&other) noexcept : buffer_(other.buffer_), parent_(other.parent_) { other.buffer_ = nullptr; other.parent_ = nullptr; } - ByteFlowSource &operator=(ByteFlowSource &&other) { + ByteFlowSource &operator=(ByteFlowSource &&other) noexcept { buffer_ = other.buffer_; parent_ = other.parent_; other.buffer_ = nullptr; @@ -187,7 +254,9 @@ class ByteFlowSource : public ByteFlowInterface { parent_ = nullptr; } void wakeup() final { - CHECK(parent_); + if (!parent_) { + return; + } parent_->wakeup(); } size_t get_need_size() final { @@ -196,19 +265,27 @@ class ByteFlowSource : public ByteFlowInterface { } return parent_->get_need_size(); } + size_t get_read_size() final { + UNREACHABLE(); + return 0; + } + size_t get_write_size() final { + CHECK(parent_); + return parent_->get_read_size(); + } private: ChainBufferReader *buffer_ = nullptr; ByteFlowInterface *parent_ = nullptr; }; -class ByteFlowSink : public ByteFlowInterface { +class ByteFlowSink final : public ByteFlowInterface { public: void set_input(ChainBufferReader *input) final { CHECK(buffer_ == nullptr); buffer_ = input; } - void set_parent(ByteFlowInterface &parent) final { + void set_parent(ByteFlowInterface & /*parent*/) final { UNREACHABLE(); } void close_input(Status status) final { @@ -224,6 +301,14 @@ class ByteFlowSink : public ByteFlowInterface { UNREACHABLE(); return 0; } + size_t get_read_size() final { + buffer_->sync_with_writer(); + return buffer_->size(); + } + size_t get_write_size() final { + UNREACHABLE(); + return 0; + } bool is_ready() { return !active_; } @@ -244,13 +329,17 @@ class ByteFlowSink : public ByteFlowInterface { ChainBufferReader *buffer_ = nullptr; }; -class ByteFlowMoveSink : public ByteFlowInterface { +class ByteFlowMoveSink final : public ByteFlowInterface { public: + ByteFlowMoveSink() = default; + explicit ByteFlowMoveSink(ChainBufferWriter *output) { + set_output(output); + } void set_input(ChainBufferReader *input) final { CHECK(!input_); input_ = input; } - void set_parent(ByteFlowInterface &parent) final { + void set_parent(ByteFlowInterface & /*parent*/) final { UNREACHABLE(); } void close_input(Status status) final { @@ -267,6 +356,15 @@ class ByteFlowMoveSink : public ByteFlowInterface { UNREACHABLE(); return 0; } + size_t get_read_size() final { + input_->sync_with_writer(); + //TODO: must be input_->size() + output_->size() + return input_->size(); + } + size_t get_write_size() final { + UNREACHABLE(); + return 0; + } void set_output(ChainBufferWriter *output) { CHECK(!output_); output_ = output; @@ -285,4 +383,5 @@ class ByteFlowMoveSink : public ByteFlowInterface { ChainBufferReader *input_ = nullptr; ChainBufferWriter *output_ = nullptr; }; + } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/CancellationToken.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/CancellationToken.h new file mode 100644 index 0000000000..19f280655c --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/CancellationToken.h @@ -0,0 +1,71 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include +#include + +namespace td { + +namespace detail { +struct RawCancellationToken { + std::atomic is_canceled_{false}; +}; +} // namespace detail + +class CancellationToken { + public: + explicit operator bool() const noexcept { + // empty CancellationToken is never canceled + if (!token_) { + return false; + } + return token_->is_canceled_.load(std::memory_order_acquire); + } + CancellationToken() = default; + explicit CancellationToken(std::shared_ptr token) : token_(std::move(token)) { + } + + private: + std::shared_ptr token_; +}; + +class CancellationTokenSource { + public: + CancellationTokenSource() = default; + CancellationTokenSource(CancellationTokenSource &&other) noexcept : token_(std::move(other.token_)) { + } + CancellationTokenSource &operator=(CancellationTokenSource &&other) noexcept { + cancel(); + token_ = std::move(other.token_); + return *this; + } + CancellationTokenSource(const CancellationTokenSource &other) = delete; + CancellationTokenSource &operator=(const CancellationTokenSource &other) = delete; + ~CancellationTokenSource() { + cancel(); + } + + CancellationToken get_cancellation_token() { + if (!token_) { + token_ = std::make_shared(); + } + return CancellationToken(token_); + } + void cancel() { + if (!token_) { + return; + } + token_->is_canceled_.store(true, std::memory_order_release); + token_.reset(); + } + + private: + std::shared_ptr token_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ChainScheduler.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ChainScheduler.h new file mode 100644 index 0000000000..721a22e137 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ChainScheduler.h @@ -0,0 +1,376 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/algorithm.h" +#include "td/utils/common.h" +#include "td/utils/Container.h" +#include "td/utils/FlatHashSet.h" +#include "td/utils/HashTableUtils.h" +#include "td/utils/List.h" +#include "td/utils/logging.h" +#include "td/utils/optional.h" +#include "td/utils/Span.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/VectorQueue.h" + +#include +#include + +namespace td { + +struct ChainSchedulerBase { + struct TaskWithParents { + uint64 task_id{}; + vector parents; + }; +}; + +template +class ChainScheduler final : public ChainSchedulerBase { + public: + using TaskId = uint64; + using ChainId = uint64; + + TaskId create_task(Span chains, ExtraT extra = {}); + + ExtraT *get_task_extra(TaskId task_id); + + optional start_next_task(); + + void pause_task(TaskId task_id); + + void finish_task(TaskId task_id); + + void reset_task(TaskId task_id); + + template + void for_each(F &&f) { + tasks_.for_each([&f](auto, Task &task) { f(task.extra); }); + } + + template + void for_each_dependent(TaskId task_id, F &&f) { + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + FlatHashSet visited; + bool check_for_collisions = task->chains.size() > 1; + for (TaskChainInfo &task_chain_info : task->chains) { + ChainInfo &chain_info = *task_chain_info.chain_info; + chain_info.chain.foreach_child(&task_chain_info.chain_node, [&](TaskId task_id, uint64) { + if (check_for_collisions && !visited.insert(task_id).second) { + return; + } + f(task_id); + }); + } + } + + private: + struct ChainNode : ListNode { + TaskId task_id{}; + uint64 generation{}; + }; + + class Chain { + public: + void add_task(ChainNode *node) { + head_.put_back(node); + } + + optional get_first() { + if (head_.empty()) { + return {}; + } + return static_cast(*head_.get_next()).task_id; + } + + optional get_child(ChainNode *chain_node) { + if (chain_node->get_next() == head_.end()) { + return {}; + } + return static_cast(*chain_node->get_next()).task_id; + } + optional get_parent(ChainNode *chain_node) { + if (chain_node->get_prev() == head_.end()) { + return {}; + } + return static_cast(chain_node->get_prev()); + } + + void finish_task(ChainNode *node) { + node->remove(); + } + + bool empty() const { + return head_.empty(); + } + + void foreach(std::function f) const { + for (auto it = head_.begin(); it != head_.end(); it = it->get_next()) { + auto &node = static_cast(*it); + f(node.task_id, node.generation); + } + } + void foreach_child(ListNode *start_node, std::function f) const { + for (auto it = start_node; it != head_.end(); it = it->get_next()) { + auto &node = static_cast(*it); + f(node.task_id, node.generation); + } + } + + private: + ListNode head_; + }; + struct ChainInfo { + Chain chain; + uint32 active_tasks{}; + uint64 generation{1}; + }; + struct TaskChainInfo { + ChainNode chain_node; + ChainId chain_id{}; + ChainInfo *chain_info{}; + }; + struct Task { + enum class State { Pending, Active, Paused } state{State::Pending}; + vector chains; + ExtraT extra; + }; + std::unordered_map> chains_; + std::unordered_map> limited_tasks_; + Container tasks_; + VectorQueue pending_tasks_; + + void try_start_task(TaskId task_id) { + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + if (task->state != Task::State::Pending) { + return; + } + for (TaskChainInfo &task_chain_info : task->chains) { + auto o_parent = task_chain_info.chain_info->chain.get_parent(&task_chain_info.chain_node); + + if (o_parent) { + if (o_parent.value()->generation != task_chain_info.chain_info->generation) { + return; + } + } + + if (task_chain_info.chain_info->active_tasks >= 10) { + limited_tasks_[task_chain_info.chain_id] = task_id; + return; + } + } + + do_start_task(task_id, task); + } + + void do_start_task(TaskId task_id, Task *task) { + for (TaskChainInfo &task_chain_info : task->chains) { + ChainInfo &chain_info = chains_[task_chain_info.chain_id]; + chain_info.active_tasks++; + task_chain_info.chain_node.generation = chain_info.generation; + } + task->state = Task::State::Active; + + pending_tasks_.push(task_id); + for_each_child(task, [&](TaskId task_id) { try_start_task(task_id); }); + } + + template + void for_each_child(Task *task, F &&f) { + for (TaskChainInfo &task_chain_info : task->chains) { + ChainInfo &chain_info = *task_chain_info.chain_info; + auto o_child = chain_info.chain.get_child(&task_chain_info.chain_node); + if (o_child) { + f(o_child.value()); + } + } + } + + void inactivate_task(TaskId task_id, bool failed) { + LOG(DEBUG) << "Inactivate " << task_id << " " << (failed ? "failed" : "finished"); + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + bool was_active = task->state == Task::State::Active; + task->state = Task::State::Pending; + for (TaskChainInfo &task_chain_info : task->chains) { + ChainInfo &chain_info = *task_chain_info.chain_info; + if (was_active) { + chain_info.active_tasks--; + } + if (was_active && failed) { + chain_info.generation = td::max(chain_info.generation, task_chain_info.chain_node.generation + 1); + } + + auto it = limited_tasks_.find(task_chain_info.chain_id); + if (it != limited_tasks_.end()) { + auto limited_task_id = it->second; + limited_tasks_.erase(it); + if (limited_task_id != task_id) { + try_start_task_later(limited_task_id); + } + } + + auto o_first = chain_info.chain.get_first(); + if (o_first) { + auto first_task_id = o_first.unwrap(); + if (first_task_id != task_id) { + try_start_task_later(first_task_id); + } + } + } + } + + void finish_chain_task(TaskChainInfo &task_chain_info) { + auto &chain = task_chain_info.chain_info->chain; + chain.finish_task(&task_chain_info.chain_node); + if (chain.empty()) { + chains_.erase(task_chain_info.chain_id); + } + } + + vector to_start_; + + void try_start_task_later(TaskId task_id) { + LOG(DEBUG) << "Start later " << task_id; + to_start_.push_back(task_id); + } + + void flush_try_start_task() { + auto moved_to_start = std::move(to_start_); + for (auto task_id : moved_to_start) { + try_start_task(task_id); + } + CHECK(to_start_.empty()); + } + + template + friend StringBuilder &operator<<(StringBuilder &sb, ChainScheduler &scheduler); +}; + +template +typename ChainScheduler::TaskId ChainScheduler::create_task(Span chains, + ExtraT extra) { + auto task_id = tasks_.create(); + Task &task = *tasks_.get(task_id); + task.extra = std::move(extra); + task.chains = transform(chains, [&](auto chain_id) { + TaskChainInfo task_chain_info; + ChainInfo &chain_info = chains_[chain_id]; + task_chain_info.chain_id = chain_id; + task_chain_info.chain_info = &chain_info; + task_chain_info.chain_node.task_id = task_id; + task_chain_info.chain_node.generation = 0; + return task_chain_info; + }); + + for (TaskChainInfo &task_chain_info : task.chains) { + ChainInfo &chain_info = *task_chain_info.chain_info; + chain_info.chain.add_task(&task_chain_info.chain_node); + } + + try_start_task(task_id); + return task_id; +} + +// TODO: return reference +template +ExtraT *ChainScheduler::get_task_extra(ChainScheduler::TaskId task_id) { // may return nullptr + auto *task = tasks_.get(task_id); + if (task == nullptr) { + return nullptr; + } + return &task->extra; +} + +template +optional ChainScheduler::start_next_task() { + if (pending_tasks_.empty()) { + return {}; + } + auto task_id = pending_tasks_.pop(); + TaskWithParents res; + res.task_id = task_id; + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + for (TaskChainInfo &task_chain_info : task->chains) { + Chain &chain = task_chain_info.chain_info->chain; + auto o_parent = chain.get_parent(&task_chain_info.chain_node); + if (o_parent) { + res.parents.push_back(o_parent.value()->task_id); + } + } + return res; +} + +template +void ChainScheduler::finish_task(ChainScheduler::TaskId task_id) { + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + CHECK(to_start_.empty()); + + inactivate_task(task_id, false); + + for_each_child(task, [&](TaskId task_id) { try_start_task_later(task_id); }); + + for (TaskChainInfo &task_chain_info : task->chains) { + finish_chain_task(task_chain_info); + } + + tasks_.erase(task_id); + flush_try_start_task(); +} + +template +void ChainScheduler::reset_task(ChainScheduler::TaskId task_id) { + CHECK(to_start_.empty()); + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + inactivate_task(task_id, true); + try_start_task_later(task_id); + flush_try_start_task(); +} + +template +void ChainScheduler::pause_task(TaskId task_id) { + auto *task = tasks_.get(task_id); + CHECK(task != nullptr); + inactivate_task(task_id, true); + task->state = Task::State::Paused; + flush_try_start_task(); +} + +template +StringBuilder &operator<<(StringBuilder &sb, ChainScheduler &scheduler) { + // 1 print chains + sb << '\n'; + for (auto &it : scheduler.chains_) { + sb << "ChainId{" << it.first << "}"; + sb << " active_cnt = " << it.second.active_tasks; + sb << " g = " << it.second.generation; + sb << ':'; + it.second.chain.foreach( + [&](auto task_id, auto generation) { sb << ' ' << *scheduler.get_task_extra(task_id) << ':' << generation; }); + sb << '\n'; + } + scheduler.tasks_.for_each([&](auto id, auto &task) { + sb << "Task: " << task.extra; + sb << " state = " << static_cast(task.state); + for (auto &task_chain_info : task.chains) { + sb << " g = " << task_chain_info.chain_node.generation; + if (task_chain_info.chain_info->generation != task_chain_info.chain_node.generation) { + sb << " chain_g = " << task_chain_info.chain_info->generation; + } + } + sb << '\n'; + }); + return sb; +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h index 9342f45a8b..669111a2ff 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h index 718f930b8a..345f1b1f89 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -8,9 +8,7 @@ #include "td/utils/common.h" #include "td/utils/invoke.h" -#include "td/utils/logging.h" -#include #include #include #include @@ -54,8 +52,7 @@ // If delay is needed, just std::forward data to temporary storage, and std::move them when call is executed. // // -// create_immediate_closure(&Actor::func, arg1, arg2, ..., argn).run(actor) -// to_delayed_closure(std::move(immediate)).run(actor) +// create_immediate_closure(&ActorT::func, arg1, arg2, ..., argn).run(actor) namespace td { template @@ -68,22 +65,22 @@ class ImmediateClosure { friend Delayed; using ActorType = ActorT; - void run(ActorT *actor) { - mem_call_tuple(actor, func, std::move(args)); - } - // no &&. just save references as references. - explicit ImmediateClosure(FunctionT func, ArgsT... args) : func(func), args(std::forward(args)...) { + explicit ImmediateClosure(FunctionT func, ArgsT... args) : args(func, std::forward(args)...) { } private: - FunctionT func; - std::tuple args; + std::tuple args; + + public: + auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) { + return mem_call_tuple(actor, std::move(args)); + } }; template ImmediateClosure create_immediate_closure( - ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { + ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) { return ImmediateClosure(func, std::forward(args)...); } @@ -92,21 +89,11 @@ template class DelayedClosure { public: using ActorType = ActorT; - using Delayed = DelayedClosure; - void run(ActorT *actor) { - mem_call_tuple(actor, func, std::move(args)); + explicit DelayedClosure(ImmediateClosure &&other) : args(std::move(other.args)) { } - DelayedClosure clone() const { - return do_clone(*this); - } - - explicit DelayedClosure(ImmediateClosure &&other) - : func(std::move(other.func)), args(std::move(other.args)) { - } - - explicit DelayedClosure(FunctionT func, ArgsT... args) : func(func), args(std::forward(args)...) { + explicit DelayedClosure(FunctionT func, ArgsT... args) : args(func, std::forward(args)...) { } template @@ -115,53 +102,16 @@ class DelayedClosure { } private: - using ArgsStorageT = std::tuple::type...>; - - FunctionT func; - ArgsStorageT args; + std::tuple::type...> args; - template - explicit DelayedClosure(const DelayedClosure &other, - std::enable_if_t::value...>::value, int> = 0) - : func(other.func), args(other.args) { - } - - template - explicit DelayedClosure( - const DelayedClosure &other, - std::enable_if_t::value...>::value, int> = 0) { - LOG(FATAL) << "Deleted constructor"; - std::abort(); - } - - template - std::enable_if_t::value...>::value, - DelayedClosure> - do_clone(const DelayedClosure &value) const { - LOG(FATAL) << "Trying to clone DelayedClosure that contains noncopyable elements"; - std::abort(); - } - - template - std::enable_if_t::value...>::value, - DelayedClosure> - do_clone(const DelayedClosure &value) const { - return DelayedClosure(value); + public: + auto run(ActorT *actor) -> decltype(mem_call_tuple(actor, std::move(args))) { + return mem_call_tuple(actor, std::move(args)); } }; -template -typename ImmediateClosure::Delayed to_delayed_closure(ImmediateClosure &&other) { - return typename ImmediateClosure::Delayed(std::move(other)); -} - -template -DelayedClosure to_delayed_closure(DelayedClosure &&other) { - return std::move(other); -} - template -auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { +auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&...args) { return DelayedClosure(func, std::forward(args)...); } diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/CombinedLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/CombinedLog.h new file mode 100644 index 0000000000..f2fe36069e --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/CombinedLog.h @@ -0,0 +1,86 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/algorithm.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +namespace td { + +class CombinedLog final : public LogInterface { + public: + void set_first(LogInterface *first) { + first_ = first; + } + + void set_second(LogInterface *second) { + second_ = second; + } + + void set_first_verbosity_level(int new_verbosity_level) { + first_verbosity_level_ = new_verbosity_level; + } + + void set_second_verbosity_level(int new_verbosity_level) { + second_verbosity_level_ = new_verbosity_level; + } + + const LogInterface *get_first() const { + return first_; + } + + const LogInterface *get_second() const { + return second_; + } + + int get_first_verbosity_level() const { + return first_verbosity_level_; + } + + int get_second_verbosity_level() const { + return second_verbosity_level_; + } + + private: + LogInterface *first_ = nullptr; + int first_verbosity_level_ = VERBOSITY_NAME(FATAL); + LogInterface *second_ = nullptr; + int second_verbosity_level_ = VERBOSITY_NAME(FATAL); + + void do_append(int log_level, CSlice slice) final { + if (first_ && log_level <= first_verbosity_level_) { + first_->do_append(log_level, slice); + } + if (second_ && log_level <= second_verbosity_level_) { + second_->do_append(log_level, slice); + } + } + + void after_rotation() final { + if (first_) { + first_->after_rotation(); + } + if (second_) { + second_->after_rotation(); + } + } + + vector get_file_paths() final { + vector result; + if (first_) { + ::td::append(result, first_->get_file_paths()); + } + if (second_) { + ::td::append(result, second_->get_file_paths()); + } + return result; + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ConcurrentHashTable.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ConcurrentHashTable.h new file mode 100644 index 0000000000..f3f0bcfa92 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ConcurrentHashTable.h @@ -0,0 +1,322 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/HazardPointers.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread_local.h" + +#include +#include +#include + +namespace td { + +// AtomicHashArray +// Building block for other concurrent hash maps +// +// Support one operation: +// template +// bool with_value(KeyT key, bool should_create, F &&func); +// +// Finds slot for key, and call func(value) +// Creates slot if should_create is true. +// Returns true if func was called. +// +// Concurrent calls with the same key may result in concurrent calls to func(value) +// It is responsibility of the caller to handle such races. +// +// Key should already be random +// It is responsibility of the caller to provide unique random key. +// One may use injective hash function, or handle collisions in some other way. + +template +class AtomicHashArray { + public: + explicit AtomicHashArray(size_t n) : nodes_(n) { + } + struct Node { + std::atomic key{KeyT{}}; + ValueT value{}; + }; + size_t size() const { + return nodes_.size(); + } + Node &node_at(size_t i) { + return nodes_[i]; + } + static KeyT empty_key() { + return KeyT{}; + } + + template + bool with_value(KeyT key, bool should_create, F &&f) { + DCHECK(key != empty_key()); + auto pos = static_cast(key) % nodes_.size(); + auto n = td::min(td::max(static_cast(300), nodes_.size() / 16 + 2), nodes_.size()); + + for (size_t i = 0; i < n; i++) { + pos++; + if (pos >= nodes_.size()) { + pos = 0; + } + auto &node = nodes_[pos]; + while (true) { + auto node_key = node.key.load(std::memory_order_acquire); + if (node_key == empty_key()) { + if (!should_create) { + return false; + } + KeyT expected_key = empty_key(); + if (node.key.compare_exchange_strong(expected_key, key, std::memory_order_relaxed, + std::memory_order_relaxed)) { + f(node.value); + return true; + } + } else if (node_key == key) { + f(node.value); + return true; + } else { + break; + } + } + } + return false; + } + + private: + std::vector nodes_; +}; + +// Simple concurrent hash map with multiple limitations +template +class ConcurrentHashMap { + using HashMap = AtomicHashArray>; + static HazardPointers hp_; + + public: + explicit ConcurrentHashMap(size_t n = 32) { + n = 1; + hash_map_.store(make_unique(n).release()); + } + ConcurrentHashMap(const ConcurrentHashMap &) = delete; + ConcurrentHashMap &operator=(const ConcurrentHashMap &) = delete; + ConcurrentHashMap(ConcurrentHashMap &&) = delete; + ConcurrentHashMap &operator=(ConcurrentHashMap &&) = delete; + ~ConcurrentHashMap() { + unique_ptr(hash_map_.load()).reset(); + } + + static std::string get_name() { + return "ConcurrrentHashMap"; + } + + static KeyT empty_key() { + return KeyT{}; + } + static ValueT empty_value() { + return ValueT{}; + } + static ValueT migrate_value() { + return (ValueT)(1); // c-style conversion because reinterpret_cast(1) is CE in MSVC + } + + ValueT insert(KeyT key, ValueT value) { + CHECK(key != empty_key()); + CHECK(value != migrate_value()); + typename HazardPointers::Holder holder(hp_, get_thread_id(), 0); + while (true) { + auto hash_map = holder.protect(hash_map_); + if (!hash_map) { + do_migrate(nullptr); + continue; + } + + bool ok = false; + ValueT inserted_value; + hash_map->with_value(key, true, [&](auto &node_value) { + ValueT expected_value = this->empty_value(); + if (node_value.compare_exchange_strong(expected_value, value, std::memory_order_release, + std::memory_order_acquire)) { + ok = true; + inserted_value = value; + } else { + if (expected_value == this->migrate_value()) { + ok = false; + } else { + ok = true; + inserted_value = expected_value; + } + } + }); + if (ok) { + return inserted_value; + } + do_migrate(hash_map); + } + } + + ValueT find(KeyT key, ValueT value) { + typename HazardPointers::Holder holder(hp_, get_thread_id(), 0); + while (true) { + auto hash_map = holder.protect(hash_map_); + if (!hash_map) { + do_migrate(nullptr); + continue; + } + + bool has_value = hash_map->with_value( + key, false, [&](auto &node_value) { value = node_value.load(std::memory_order_acquire); }); + if (!has_value || value != migrate_value()) { + return value; + } + do_migrate(hash_map); + } + } + + template + void for_each(F &&f) { + auto hash_map = hash_map_.load(); + CHECK(hash_map); + auto size = hash_map->size(); + for (size_t i = 0; i < size; i++) { + auto &node = hash_map->node_at(i); + auto key = node.key.load(std::memory_order_relaxed); + auto value = node.value.load(std::memory_order_relaxed); + + if (key != empty_key()) { + CHECK(value != migrate_value()); + if (value != empty_value()) { + f(key, value); + } + } + } + } + + private: + // use no padding intentionally + std::atomic hash_map_{nullptr}; + + std::mutex migrate_mutex_; + std::condition_variable migrate_cv_; + + int migrate_cnt_{0}; + int migrate_generation_{0}; + HashMap *migrate_from_hash_map_{nullptr}; + HashMap *migrate_to_hash_map_{nullptr}; + struct Task { + size_t begin; + size_t end; + bool empty() const { + return begin >= end; + } + size_t size() const { + if (empty()) { + return 0; + } + return end - begin; + } + }; + + struct TaskCreator { + size_t chunk_size; + size_t size; + std::atomic pos{0}; + Task create() { + auto i = pos++; + auto begin = i * chunk_size; + auto end = begin + chunk_size; + if (end > size) { + end = size; + } + return {begin, end}; + } + }; + TaskCreator task_creator; + + void do_migrate(HashMap *ptr) { + //LOG(ERROR) << "In do_migrate: " << ptr; + std::unique_lock lock(migrate_mutex_); + if (hash_map_.load() != ptr) { + return; + } + init_migrate(); + CHECK(!ptr || migrate_from_hash_map_ == ptr); + migrate_cnt_++; + auto migrate_generation = migrate_generation_; + lock.unlock(); + + run_migrate(); + + lock.lock(); + migrate_cnt_--; + if (migrate_cnt_ == 0) { + finish_migrate(); + } + migrate_cv_.wait(lock, [&] { return migrate_generation_ != migrate_generation; }); + } + + void finish_migrate() { + //LOG(ERROR) << "In finish_migrate"; + hash_map_.store(migrate_to_hash_map_); + hp_.retire(get_thread_id(), migrate_from_hash_map_); + migrate_from_hash_map_ = nullptr; + migrate_to_hash_map_ = nullptr; + migrate_generation_++; + migrate_cv_.notify_all(); + } + + void init_migrate() { + if (migrate_from_hash_map_ != nullptr) { + return; + } + //LOG(ERROR) << "In init_migrate"; + CHECK(migrate_cnt_ == 0); + migrate_generation_++; + migrate_from_hash_map_ = hash_map_.exchange(nullptr); + auto new_size = migrate_from_hash_map_->size() * 2; + migrate_to_hash_map_ = make_unique(new_size).release(); + task_creator.chunk_size = 100; + task_creator.size = migrate_from_hash_map_->size(); + task_creator.pos = 0; + } + + void run_migrate() { + //LOG(ERROR) << "In run_migrate"; + size_t cnt = 0; + while (true) { + auto task = task_creator.create(); + cnt += task.size(); + if (task.empty()) { + break; + } + run_task(task); + } + //LOG(ERROR) << "In run_migrate " << cnt; + } + + void run_task(Task task) { + for (auto i = task.begin; i < task.end; i++) { + auto &node = migrate_from_hash_map_->node_at(i); + auto old_value = node.value.exchange(migrate_value(), std::memory_order_acq_rel); + if (old_value == 0) { + continue; + } + auto node_key = node.key.load(std::memory_order_relaxed); + //LOG(ERROR) << node_key << " " << node_key; + auto ok = migrate_to_hash_map_->with_value( + node_key, true, [&](auto &node_value) { node_value.store(old_value, std::memory_order_relaxed); }); + LOG_CHECK(ok) << "Migration overflow"; + } + } +}; + +template +HazardPointers::HashMap> ConcurrentHashMap::hp_(64); + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h index 57b4bb4d16..418edfd56a 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,7 +7,6 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/logging.h" #include @@ -107,8 +106,8 @@ class Container { } int32 decode_id(Id id) const { - int32 slot_id = static_cast(id >> 32); - uint32 generation = static_cast(id); + auto slot_id = static_cast(id >> 32); + auto generation = static_cast(id); if (slot_id < 0 || slot_id >= static_cast(slots_.size())) { return -1; } diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Context.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Context.h new file mode 100644 index 0000000000..e9cac2ed09 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Context.h @@ -0,0 +1,44 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/thread_local.h" + +namespace td { + +template +class Context { + public: + static Impl *get() { + return context_; + } + class Guard { + public: + explicit Guard(Impl *new_context) { + old_context_ = context_; + context_ = new_context; + } + ~Guard() { + context_ = old_context_; + } + Guard(const Guard &) = delete; + Guard &operator=(const Guard &) = delete; + Guard(Guard &&) = delete; + Guard &operator=(Guard &&) = delete; + + private: + Impl *old_context_; + }; + + private: + static TD_THREAD_LOCAL Impl *context_; +}; + +template +TD_THREAD_LOCAL Impl *Context::context_; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/DecTree.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/DecTree.h new file mode 100644 index 0000000000..6044842f69 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/DecTree.h @@ -0,0 +1,216 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Random.h" + +#include +#include + +namespace td { + +template > +class DecTree { + struct Node { + unique_ptr left_; + unique_ptr right_; + size_t size_; + KeyType key_; + ValueType value_; + uint32 y_; + + void relax() { + size_ = 1; + if (left_ != nullptr) { + size_ += left_->size_; + } + if (right_ != nullptr) { + size_ += right_->size_; + } + } + + Node(KeyType key, ValueType value, uint32 y) : size_(1), key_(std::move(key)), value_(std::move(value)), y_(y) { + } + }; + unique_ptr root_; + + static unique_ptr create_node(KeyType key, ValueType value, uint32 y) { + return make_unique(std::move(key), std::move(value), y); + } + + static unique_ptr insert_node(unique_ptr Tree, KeyType key, ValueType value, uint32 y) { + if (Tree == nullptr) { + return create_node(std::move(key), std::move(value), y); + } + if (Tree->y_ < y) { + auto P = split_node(std::move(Tree), key); + auto T = create_node(std::move(key), std::move(value), y); + T->left_ = std::move(P.first); + T->right_ = std::move(P.second); + T->relax(); + return T; + } + if (Compare()(key, Tree->key_)) { + Tree->left_ = insert_node(std::move(Tree->left_), std::move(key), std::move(value), y); + } else if (Compare()(Tree->key_, key)) { + Tree->right_ = insert_node(std::move(Tree->right_), std::move(key), std::move(value), y); + } else { + // ?? assert + } + Tree->relax(); + return Tree; + } + + static unique_ptr remove_node(unique_ptr Tree, const KeyType &key) { + if (Tree == nullptr) { + // ?? assert + return nullptr; + } + if (Compare()(key, Tree->key_)) { + Tree->left_ = remove_node(std::move(Tree->left_), key); + } else if (Compare()(Tree->key_, key)) { + Tree->right_ = remove_node(std::move(Tree->right_), key); + } else { + Tree = merge_node(std::move(Tree->left_), std::move(Tree->right_)); + } + if (Tree != nullptr) { + Tree->relax(); + } + return Tree; + } + + static ValueType *get_node(unique_ptr &Tree, const KeyType &key) { + if (Tree == nullptr) { + return nullptr; + } + if (Compare()(key, Tree->key_)) { + return get_node(Tree->left_, key); + } else if (Compare()(Tree->key_, key)) { + return get_node(Tree->right_, key); + } else { + return &Tree->value_; + } + } + + static ValueType *get_node_by_idx(unique_ptr &Tree, size_t idx) { + CHECK(Tree != nullptr); + auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0; + if (idx < s) { + return get_node_by_idx(Tree->left_, idx); + } else if (idx == s) { + return &Tree->value_; + } else { + return get_node_by_idx(Tree->right_, idx - s - 1); + } + } + + static const ValueType *get_node(const unique_ptr &Tree, const KeyType &key) { + if (Tree == nullptr) { + return nullptr; + } + if (Compare()(key, Tree->key_)) { + return get_node(Tree->left_, key); + } else if (Compare()(Tree->key_, key)) { + return get_node(Tree->right_, key); + } else { + return &Tree->value_; + } + } + + static const ValueType *get_node_by_idx(const unique_ptr &Tree, size_t idx) { + CHECK(Tree != nullptr); + auto s = (Tree->left_ != nullptr) ? Tree->left_->size_ : 0; + if (idx < s) { + return get_node_by_idx(Tree->left_, idx); + } else if (idx == s) { + return &Tree->value_; + } else { + return get_node_by_idx(Tree->right_, idx - s - 1); + } + } + + static std::pair, unique_ptr> split_node(unique_ptr Tree, const KeyType &key) { + if (Tree == nullptr) { + return {nullptr, nullptr}; + } + if (Compare()(key, Tree->key_)) { + auto P = split_node(std::move(Tree->left_), key); + Tree->left_ = std::move(P.second); + Tree->relax(); + P.second = std::move(Tree); + return P; + } else { + auto P = split_node(std::move(Tree->right_), key); + Tree->right_ = std::move(P.first); + Tree->relax(); + P.first = std::move(Tree); + return P; + } + } + + static unique_ptr merge_node(unique_ptr left, unique_ptr right) { + if (left == nullptr) { + return right; + } + if (right == nullptr) { + return left; + } + if (left->y_ < right->y_) { + right->left_ = merge_node(std::move(left), std::move(right->left_)); + right->relax(); + return right; + } else { + left->right_ = merge_node(std::move(left->right_), std::move(right)); + left->relax(); + return left; + } + } + + public: + size_t size() const { + if (root_ == nullptr) { + return 0; + } else { + return root_->size_; + } + } + void insert(KeyType key, ValueType value) { + root_ = insert_node(std::move(root_), std::move(key), std::move(value), Random::fast_uint32()); + } + void remove(const KeyType &key) { + root_ = remove_node(std::move(root_), key); + } + void reset() { + root_ = nullptr; + } + ValueType *get(const KeyType &key) { + return get_node(root_, key); + } + ValueType *get_random() { + if (size() == 0) { + return nullptr; + } else { + return get_node_by_idx(root_, Random::fast_uint32() % size()); + } + } + const ValueType *get(const KeyType &key) const { + return get_node(root_, key); + } + const ValueType *get_random() const { + if (size() == 0) { + return nullptr; + } else { + return get_node_by_idx(root_, Random::fast_uint32() % size()); + } + } + bool exists(const KeyType &key) const { + return get_node(root_, key) != nullptr; + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Destructor.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Destructor.h new file mode 100644 index 0000000000..ced4a8eca8 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Destructor.h @@ -0,0 +1,52 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include +#include + +namespace td { + +class Destructor { + public: + Destructor() = default; + Destructor(const Destructor &other) = delete; + Destructor &operator=(const Destructor &other) = delete; + Destructor(Destructor &&other) = default; + Destructor &operator=(Destructor &&other) = default; + virtual ~Destructor() = default; +}; + +template +class LambdaDestructor final : public Destructor { + public: + explicit LambdaDestructor(F &&f) : f_(std::move(f)) { + } + LambdaDestructor(const LambdaDestructor &other) = delete; + LambdaDestructor &operator=(const LambdaDestructor &other) = delete; + LambdaDestructor(LambdaDestructor &&other) = default; + LambdaDestructor &operator=(LambdaDestructor &&other) = default; + ~LambdaDestructor() final { + f_(); + } + + private: + F f_; +}; + +template +auto create_destructor(F &&f) { + return make_unique>(std::forward(f)); +} +template +auto create_shared_destructor(F &&f) { + return std::make_shared>(std::forward(f)); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h index ca7c0493ff..367bc9fa8c 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,9 +7,9 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/logging.h" -#include "td/utils/misc.h" +#include "td/utils/WaitFreeVector.h" +#include #include #include @@ -21,7 +21,8 @@ class Enumerator { using Key = int32; Key add(ValueT v) { - int32 next_id = narrow_cast(arr_.size() + 1); + CHECK(arr_.size() < static_cast(std::numeric_limits::max() - 1)); + auto next_id = static_cast(arr_.size() + 1); bool was_inserted; decltype(map_.begin()) it; std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id); @@ -37,9 +38,18 @@ class Enumerator { return *arr_[pos]; } + size_t size() const { + CHECK(map_.size() == arr_.size()); + return arr_.size(); + } + + bool empty() const { + return size() == 0; + } + private: std::map map_; - std::vector arr_; + WaitFreeVector arr_; }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/EpochBasedMemoryReclamation.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/EpochBasedMemoryReclamation.h new file mode 100644 index 0000000000..a11d307672 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/EpochBasedMemoryReclamation.h @@ -0,0 +1,201 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/sleep.h" + +#include +#include + +namespace td { + +template +class EpochBasedMemoryReclamation { + public: + EpochBasedMemoryReclamation(const EpochBasedMemoryReclamation &other) = delete; + EpochBasedMemoryReclamation &operator=(const EpochBasedMemoryReclamation &other) = delete; + EpochBasedMemoryReclamation(EpochBasedMemoryReclamation &&other) = delete; + EpochBasedMemoryReclamation &operator=(EpochBasedMemoryReclamation &&other) = delete; + ~EpochBasedMemoryReclamation() = default; + + class Locker { + public: + Locker(size_t thread_id, EpochBasedMemoryReclamation *ebmr) : thread_id_(thread_id), ebmr_(ebmr) { + } + Locker(const Locker &other) = delete; + Locker &operator=(const Locker &other) = delete; + Locker(Locker &&other) = default; + Locker &operator=(Locker &&other) = delete; + + ~Locker() { + if (ebmr_) { + retire_sync(); + unlock(); + (void)ebmr_.release(); + } + } + void lock() { + DCHECK(ebmr_); + ebmr_->lock(thread_id_); + } + void unlock() { + DCHECK(ebmr_); + ebmr_->unlock(thread_id_); + } + + void retire_sync() { + ebmr_->retire_sync(thread_id_); + } + + void retire() { + ebmr_->retire(thread_id_); + } + + void retire(T *ptr) { + ebmr_->retire(thread_id_, ptr); + } + + private: + size_t thread_id_; + struct Never { + template + void operator()(S *) const { + UNREACHABLE(); + } + }; + std::unique_ptr ebmr_; + }; + + explicit EpochBasedMemoryReclamation(size_t threads_n) : threads_(threads_n) { + } + + Locker get_locker(size_t thread_id) { + return Locker{thread_id, this}; + } + + size_t to_delete_size_unsafe() const { + size_t res = 0; + for (auto &thread_data : threads_) { + // LOG(ERROR) << "---" << thread_data.epoch.load() / 2; + for (size_t i = 0; i < MAX_BAGS; i++) { + res += thread_data.to_delete[i].size(); + // LOG(ERROR) << thread_data.to_delete[i].size(); + } + } + return res; + } + + private: + static constexpr size_t MAX_BAGS = 3; + struct ThreadData { + std::atomic epoch{1}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + + size_t to_skip{0}; + size_t checked_thread_i{0}; + size_t bag_i{0}; + std::vector> to_delete[MAX_BAGS]; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector>) * MAX_BAGS]; + + void rotate_bags() { + bag_i = (bag_i + 1) % MAX_BAGS; + to_delete[bag_i].clear(); + } + + void set_epoch(int64 new_epoch) { + //LOG(ERROR) << new_epoch; + if (epoch.load(std::memory_order_relaxed) / 2 != new_epoch) { + checked_thread_i = 0; + to_skip = 0; + rotate_bags(); + } + epoch = new_epoch * 2; + } + + void idle() { + epoch.store(epoch.load(std::memory_order_relaxed) | 1); + } + + size_t undeleted() const { + size_t res = 0; + for (size_t i = 0; i < MAX_BAGS; i++) { + res += to_delete[i].size(); + } + return res; + } + }; + std::vector threads_; + char pad[TD_CONCURRENCY_PAD - sizeof(std::vector)]; + + std::atomic epoch_{1}; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + + void lock(size_t thread_id) { + auto &data = threads_[thread_id]; + auto epoch = epoch_.load(); + data.set_epoch(epoch); + + if (data.to_skip == 0) { + data.to_skip = 30; + step_check(data); + } else { + data.to_skip--; + } + } + + void unlock(size_t thread_id) { + //LOG(ERROR) << "UNLOCK"; + auto &data = threads_[thread_id]; + data.idle(); + } + + bool step_check(ThreadData &data) { + auto epoch = data.epoch.load(std::memory_order_relaxed) / 2; + auto checked_thread_epoch = threads_[data.checked_thread_i].epoch.load(); + if (checked_thread_epoch % 2 == 1 || checked_thread_epoch / 2 == epoch) { + data.checked_thread_i++; + if (data.checked_thread_i == threads_.size()) { + if (epoch_.compare_exchange_strong(epoch, epoch + 1)) { + data.set_epoch(epoch + 1); + } else { + data.set_epoch(epoch); + } + } + return true; + } + return false; + } + + void retire_sync(size_t thread_id) { + auto &data = threads_[thread_id]; + + while (true) { + retire(thread_id); + data.idle(); + if (data.undeleted() == 0) { + break; + } + usleep_for(1000); + } + } + + void retire(size_t thread_id) { + auto &data = threads_[thread_id]; + data.set_epoch(epoch_.load()); + while (step_check(data) && data.undeleted() != 0) { + } + } + + void retire(size_t thread_id, T *ptr) { + auto &data = threads_[thread_id]; + data.to_delete[data.bag_i].push_back(unique_ptr{ptr}); + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.cpp new file mode 100644 index 0000000000..3a410bd0d5 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.cpp @@ -0,0 +1,20 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/ExitGuard.h" + +#include "td/utils/logging.h" + +namespace td { + +std::atomic ExitGuard::is_exited_{false}; + +ExitGuard::~ExitGuard() { + is_exited_.store(true, std::memory_order_relaxed); + set_verbosity_level(VERBOSITY_NAME(FATAL)); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.h new file mode 100644 index 0000000000..dd721fb5b9 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ExitGuard.h @@ -0,0 +1,30 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include + +namespace td { + +class ExitGuard { + public: + ExitGuard() = default; + ExitGuard(ExitGuard &&) = delete; + ExitGuard &operator=(ExitGuard &&) = delete; + ExitGuard(const ExitGuard &) = delete; + ExitGuard &operator=(const ExitGuard &) = delete; + ~ExitGuard(); + + static bool is_exited() { + return is_exited_.load(std::memory_order_relaxed); + } + + private: + static std::atomic is_exited_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp index e3c84f1713..a16731442a 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -8,85 +8,133 @@ #include "td/utils/common.h" #include "td/utils/logging.h" -#include "td/utils/port/Fd.h" #include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/port/thread_local.h" #include "td/utils/Slice.h" - -#include +#include "td/utils/SliceBuilder.h" +#include "td/utils/Time.h" namespace td { -bool FileLog::init(string path, int64 rotate_threshold) { +Status FileLog::init(string path, int64 rotate_threshold, bool redirect_stderr) { + if (path.empty()) { + return Status::Error("Log file path must be non-empty"); + } if (path == path_) { set_rotate_threshold(rotate_threshold); - return true; + return Status::OK(); } - auto r_fd = FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append); - if (r_fd.is_error()) { - LOG(ERROR) << "Can't open log: " << r_fd.error(); - return false; - } + TRY_RESULT(fd, FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append)); fd_.close(); - fd_ = r_fd.move_as_ok(); - Fd::duplicate(fd_.get_fd(), Fd::Stderr()).ignore(); + fd_ = std::move(fd); + if (!Stderr().empty() && redirect_stderr) { + fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore(); + } - path_ = std::move(path); - size_ = fd_.get_size(); + auto r_path = realpath(path, true); + if (r_path.is_error()) { + path_ = std::move(path); + } else { + path_ = r_path.move_as_ok(); + } + TRY_RESULT_ASSIGN(size_, fd_.get_size()); rotate_threshold_ = rotate_threshold; - return true; + redirect_stderr_ = redirect_stderr; + return Status::OK(); +} + +Slice FileLog::get_path() const { + return path_; +} + +vector FileLog::get_file_paths() { + vector result; + if (!path_.empty()) { + result.push_back(path_); + result.push_back(PSTRING() << path_ << ".old"); + } + return result; } void FileLog::set_rotate_threshold(int64 rotate_threshold) { rotate_threshold_ = rotate_threshold; } -void FileLog::append(CSlice cslice, int log_level) { - Slice slice = cslice; +int64 FileLog::get_rotate_threshold() const { + return rotate_threshold_; +} + +bool FileLog::get_redirect_stderr() const { + return redirect_stderr_; +} + +void FileLog::do_append(int log_level, CSlice slice) { + auto start_time = Time::now(); + if (size_ > rotate_threshold_ || want_rotate_.load(std::memory_order_relaxed)) { + auto status = rename(path_, PSLICE() << path_ << ".old"); + if (status.is_error()) { + process_fatal_error(PSLICE() << status << " in " << __FILE__ << " at " << __LINE__ << '\n'); + } + do_after_rotation(); + } while (!slice.empty()) { + if (redirect_stderr_) { + while (has_log_guard()) { + // spin + } + } auto r_size = fd_.write(slice); if (r_size.is_error()) { - process_fatal_error(r_size.error().message()); + process_fatal_error(PSLICE() << r_size.error() << " in " << __FILE__ << " at " << __LINE__ << '\n'); } auto written = r_size.ok(); size_ += static_cast(written); slice.remove_prefix(written); } - if (log_level == VERBOSITY_NAME(FATAL)) { - process_fatal_error(cslice); - } - - if (size_ > rotate_threshold_) { - auto status = rename(path_, path_ + ".old"); - if (status.is_error()) { - process_fatal_error(status.message()); - } - do_rotate(); + auto total_time = Time::now() - start_time; + if (total_time >= 0.1 && log_level >= 1) { + auto thread_id = get_thread_id(); + auto r_size = fd_.write(PSLICE() << "[ 1][t" << (0 <= thread_id && thread_id < 10 ? " " : "") << thread_id + << "] !!! Previous logging took " << total_time << " seconds !!!\n"); + r_size.ignore(); } } -void FileLog::rotate() { +void FileLog::after_rotation() { if (path_.empty()) { return; } - do_rotate(); + do_after_rotation(); +} + +void FileLog::lazy_rotate() { + want_rotate_ = true; } -void FileLog::do_rotate() { - auto current_verbosity_level = GET_VERBOSITY_LEVEL(); - SET_VERBOSITY_LEVEL(std::numeric_limits::min()); // to ensure that nothing will be printed to the closed log +void FileLog::do_after_rotation() { + want_rotate_ = false; + ScopedDisableLog disable_log; // to ensure that nothing will be printed to the closed log CHECK(!path_.empty()); fd_.close(); - auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Truncate | FileFd::Write); + auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Write | FileFd::Append); if (r_fd.is_error()) { - process_fatal_error(r_fd.error().message()); + process_fatal_error(PSLICE() << r_fd.error() << " in " << __FILE__ << " at " << __LINE__ << '\n'); } fd_ = r_fd.move_as_ok(); - Fd::duplicate(fd_.get_fd(), Fd::Stderr()).ignore(); + if (!Stderr().empty() && redirect_stderr_) { + fd_.get_native_fd().duplicate(Stderr().get_native_fd()).ignore(); + } size_ = 0; - SET_VERBOSITY_LEVEL(current_verbosity_level); +} + +Result> FileLog::create(string path, int64 rotate_threshold, bool redirect_stderr) { + auto l = make_unique(); + TRY_STATUS(l->init(std::move(path), rotate_threshold, redirect_stderr)); + return std::move(l); } } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h index 12e9d1479a..ad4ec5eb02 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -10,28 +10,45 @@ #include "td/utils/logging.h" #include "td/utils/port/FileFd.h" #include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include namespace td { -class FileLog : public LogInterface { +class FileLog final : public LogInterface { static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20); public: - bool init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD); + static Result> create(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, + bool redirect_stderr = true); + Status init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD, bool redirect_stderr = true); + + Slice get_path() const; + + vector get_file_paths() final; void set_rotate_threshold(int64 rotate_threshold); - void append(CSlice cslice, int log_level) override; + int64 get_rotate_threshold() const; - void rotate() override; + bool get_redirect_stderr() const; + + void after_rotation() final; + + void lazy_rotate(); private: FileFd fd_; string path_; - int64 size_; - int64 rotate_threshold_; + int64 size_ = 0; + int64 rotate_threshold_ = 0; + bool redirect_stderr_ = false; + std::atomic want_rotate_{false}; + + void do_append(int log_level, CSlice slice) final; - void do_rotate(); + void do_after_rotation(); }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMap.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMap.h new file mode 100644 index 0000000000..51aa6d3e4c --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMap.h @@ -0,0 +1,24 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +//#include "td/utils/FlatHashMapChunks.h" +#include "td/utils/FlatHashTable.h" +#include "td/utils/HashTableUtils.h" +#include "td/utils/MapNode.h" + +#include +//#include + +namespace td { + +template , class EqT = std::equal_to> +using FlatHashMap = FlatHashTable, HashT, EqT>; +//using FlatHashMap = FlatHashMapChunks; +//using FlatHashMap = std::unordered_map; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMapChunks.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMapChunks.h new file mode 100644 index 0000000000..6df4842bc1 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashMapChunks.h @@ -0,0 +1,575 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/bits.h" +#include "td/utils/common.h" +#include "td/utils/fixed_vector.h" +#include "td/utils/HashTableUtils.h" +#include "td/utils/MapNode.h" +#include "td/utils/SetNode.h" + +#include +#include +#include +#include +#include +#include + +#if defined(__SSE2__) || (TD_MSVC && (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) +#define TD_SSE2 1 +#endif + +#ifdef __aarch64__ +#include +#endif + +#if TD_SSE2 +#include +#endif + +namespace td { +template +struct MaskIterator { + uint64 mask; + explicit operator bool() const noexcept { + return mask != 0; + } + int pos() const { + return count_trailing_zeroes64(mask) / shift; + } + void next() { + mask &= mask - 1; + } + + // For foreach + bool operator!=(MaskIterator &other) const { + return mask != other.mask; + } + auto operator*() const { + return pos(); + } + void operator++() { + next(); + } + auto begin() { + return *this; + } + auto end() { + return MaskIterator{0u}; + } +}; + +struct MaskPortable { + static MaskIterator<1> equal_mask(uint8 *bytes, uint8 needle) { + uint64 res = 0; + for (int i = 0; i < 16; i++) { + res |= (bytes[i] == needle) << i; + } + return {res & ((1u << 14) - 1)}; + } +}; + +#ifdef __aarch64__ +struct MaskNeonFolly { + static MaskIterator<4> equal_mask(uint8 *bytes, uint8 needle) { + uint8x16_t input_mask = vld1q_u8(bytes); + auto needle_mask = vdupq_n_u8(needle); + auto eq_mask = vceqq_u8(input_mask, needle_mask); + // get info from every byte into the bottom half of every uint16 + // by shifting right 4, then round to get it into a 64-bit vector + uint8x8_t shifted_eq_mask = vshrn_n_u16(vreinterpretq_u16_u8(eq_mask), 4); + uint64 mask = vget_lane_u64(vreinterpret_u64_u8(shifted_eq_mask), 0); + return {mask & 0x11111111111111}; + } +}; + +struct MaskNeon { + static MaskIterator<1> equal_mask(uint8 *bytes, uint8 needle) { + uint8x16_t input_mask = vld1q_u8(bytes); + auto needle_mask = vdupq_n_u8(needle); + auto eq_mask = vceqq_u8(input_mask, needle_mask); + uint16x8_t MASK = vdupq_n_u16(0x180); + uint16x8_t a_masked = vandq_u16(vreinterpretq_u16_u8(eq_mask), MASK); + const int16 __attribute__((aligned(16))) SHIFT_ARR[8] = {-7, -5, -3, -1, 1, 3, 5, 7}; + int16x8_t SHIFT = vld1q_s16(SHIFT_ARR); + uint16x8_t a_shifted = vshlq_u16(a_masked, SHIFT); + return {vaddvq_u16(a_shifted) & ((1u << 14) - 1)}; + } +}; +#elif TD_SSE2 +struct MaskSse2 { + static MaskIterator<1> equal_mask(uint8 *bytes, uint8 needle) { + auto input_mask = _mm_loadu_si128(reinterpret_cast(bytes)); + auto needle_mask = _mm_set1_epi8(needle); + auto match_mask = _mm_cmpeq_epi8(needle_mask, input_mask); + return {static_cast(_mm_movemask_epi8(match_mask)) & ((1u << 14) - 1)}; + } +}; +#endif + +#ifdef __aarch64__ +using MaskHelper = MaskNeonFolly; +#elif TD_SSE2 +using MaskHelper = MaskSse2; +#else +using MaskHelper = MaskPortable; +#endif + +template +class FlatHashTableChunks { + public: + using Self = FlatHashTableChunks; + using Node = NodeT; + using NodeIterator = typename fixed_vector::iterator; + using ConstNodeIterator = typename fixed_vector::const_iterator; + + using KeyT = typename Node::public_key_type; + using key_type = typename Node::public_key_type; + using value_type = typename Node::public_type; + + struct Iterator { + using iterator_category = std::bidirectional_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = FlatHashTableChunks::value_type; + using pointer = value_type *; + using reference = value_type &; + + friend class FlatHashTableChunks; + Iterator &operator++() { + do { + ++it_; + } while (it_ != map_->nodes_.end() && it_->empty()); + return *this; + } + Iterator &operator--() { + do { + --it_; + } while (it_->empty()); + return *this; + } + reference operator*() { + return it_->get_public(); + } + pointer operator->() { + return &it_->get_public(); + } + bool operator==(const Iterator &other) const { + DCHECK(map_ == other.map_); + return it_ == other.it_; + } + bool operator!=(const Iterator &other) const { + DCHECK(map_ == other.map_); + return it_ != other.it_; + } + + Iterator() = default; + Iterator(NodeIterator it, Self *map) : it_(std::move(it)), map_(map) { + } + + private: + NodeIterator it_; + Self *map_; + }; + + struct ConstIterator { + using iterator_category = std::bidirectional_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = FlatHashTableChunks::value_type; + using pointer = const value_type *; + using reference = const value_type &; + + friend class FlatHashTableChunks; + ConstIterator &operator++() { + ++it_; + return *this; + } + ConstIterator &operator--() { + --it_; + return *this; + } + reference operator*() { + return *it_; + } + pointer operator->() { + return &*it_; + } + bool operator==(const ConstIterator &other) const { + return it_ == other.it_; + } + bool operator!=(const ConstIterator &other) const { + return it_ != other.it_; + } + + ConstIterator() = default; + ConstIterator(Iterator it) : it_(std::move(it)) { + } + + private: + Iterator it_; + }; + using iterator = Iterator; + using const_iterator = ConstIterator; + + FlatHashTableChunks() = default; + FlatHashTableChunks(const FlatHashTableChunks &other) { + assign(other); + } + FlatHashTableChunks &operator=(const FlatHashTableChunks &other) { + clear(); + assign(other); + return *this; + } + + FlatHashTableChunks(std::initializer_list nodes) { + reserve(nodes.size()); + for (auto &new_node : nodes) { + CHECK(!new_node.empty()); + if (count(new_node.key()) > 0) { + continue; + } + Node node; + node.copy_from(new_node); + emplace_node(std::move(node)); + } + } + + FlatHashTableChunks(FlatHashTableChunks &&other) noexcept { + swap(other); + } + FlatHashTableChunks &operator=(FlatHashTableChunks &&other) noexcept { + swap(other); + return *this; + } + void swap(FlatHashTableChunks &other) noexcept { + nodes_.swap(other.nodes_); + chunks_.swap(other.chunks_); + std::swap(used_nodes_, other.used_nodes_); + } + ~FlatHashTableChunks() = default; + + size_t bucket_count() const { + return nodes_.size(); + } + + Iterator find(const KeyT &key) { + if (empty() || is_hash_table_key_empty(key)) { + return end(); + } + const auto hash = calc_hash(key); + auto chunk_it = get_chunk_it(hash.chunk_i); + while (true) { + auto chunk_i = chunk_it.pos(); + auto chunk_begin = nodes_.begin() + chunk_i * Chunk::CHUNK_SIZE; + //__builtin_prefetch(chunk_begin); + auto &chunk = chunks_[chunk_i]; + auto mask_it = MaskHelper::equal_mask(chunk.ctrl, hash.small_hash); + for (auto pos : mask_it) { + auto it = chunk_begin + pos; + if (likely(EqT()(it->key(), key))) { + return Iterator{it, this}; + } + } + if (chunk.skipped_cnt == 0) { + break; + } + chunk_it.next(); + } + return end(); + } + + ConstIterator find(const KeyT &key) const { + return ConstIterator(const_cast(this)->find(key)); + } + + size_t size() const { + return used_nodes_; + } + + bool empty() const { + return size() == 0; + } + + Iterator begin() { + if (empty()) { + return end(); + } + auto it = nodes_.begin(); + while (it->empty()) { + ++it; + } + return Iterator(it, this); + } + Iterator end() { + return Iterator(nodes_.end(), this); + } + + ConstIterator begin() const { + return ConstIterator(const_cast(this)->begin()); + } + ConstIterator end() const { + return ConstIterator(const_cast(this)->end()); + } + + void reserve(size_t size) { + //size_t want_size = normalize(size * 5 / 3 + 1); + size_t want_size = normalize(size * 14 / 12 + 1); + // size_t want_size = size * 2; + if (want_size > nodes_.size()) { + resize(want_size); + } + } + + template + std::pair emplace(KeyT key, ArgsT &&...args) { + CHECK(!is_hash_table_key_empty(key)); + auto it = find(key); + if (it != end()) { + return {it, false}; + } + try_grow(); + + auto hash = calc_hash(key); + auto chunk_it = get_chunk_it(hash.chunk_i); + while (true) { + auto chunk_i = chunk_it.pos(); + auto &chunk = chunks_[chunk_i]; + auto mask_it = MaskHelper::equal_mask(chunk.ctrl, 0); + if (mask_it) { + auto shift = mask_it.pos(); + DCHECK(chunk.ctrl[shift] == 0); + auto node_it = nodes_.begin() + shift + chunk_i * Chunk::CHUNK_SIZE; + DCHECK(node_it->empty()); + node_it->emplace(std::move(key), std::forward(args)...); + DCHECK(!node_it->empty()); + chunk.ctrl[shift] = hash.small_hash; + used_nodes_++; + return {{node_it, this}, true}; + } + CHECK(chunk.skipped_cnt != std::numeric_limits::max()); + chunk.skipped_cnt++; + chunk_it.next(); + } + } + + std::pair insert(KeyT key) { + return emplace(std::move(key)); + } + + template + void insert(ItT begin, ItT end) { + for (; begin != end; ++begin) { + emplace(*begin); + } + } + + template + T &operator[](const KeyT &key) { + return emplace(key).first->second; + } + + size_t erase(const KeyT &key) { + auto it = find(key); + if (it == end()) { + return 0; + } + erase(it); + try_shrink(); + return 1; + } + + size_t count(const KeyT &key) const { + return find(key) != end(); + } + + void clear() { + used_nodes_ = 0; + nodes_ = {}; + chunks_ = {}; + } + + void erase(Iterator it) { + DCHECK(it != end()); + DCHECK(!it.it_->empty()); + erase_node(it.it_); + } + + template + void remove_if(F &&f) { + for (auto it = nodes_.begin(), end = nodes_.end(); it != end; ++it) { + if (!it->empty() && f(it->get_public())) { + erase_node(it); + } + } + try_shrink(); + } + + private: + struct Chunk { + static constexpr int CHUNK_SIZE = 14; + static constexpr int MASK = (1 << CHUNK_SIZE) - 1; + // 0x0 - empty + uint8 ctrl[CHUNK_SIZE] = {}; + uint16 skipped_cnt{0}; + }; + fixed_vector nodes_; + fixed_vector chunks_; + size_t used_nodes_{}; + + void assign(const FlatHashTableChunks &other) { + reserve(other.size()); + for (const auto &new_node : other) { + Node node; + node.copy_from(new_node); + emplace_node(std::move(node)); + } + } + + void try_grow() { + if (should_grow(used_nodes_ + 1, nodes_.size())) { + grow(); + } + } + static bool should_grow(size_t used_count, size_t bucket_count) { + return used_count * 14 > bucket_count * 12; + } + void try_shrink() { + if (should_shrink(used_nodes_, nodes_.size())) { + shrink(); + } + } + static bool should_shrink(size_t used_count, size_t bucket_count) { + return used_count * 10 < bucket_count; + } + + static size_t normalize(size_t size) { + auto x = (size / Chunk::CHUNK_SIZE) | 1; + auto y = static_cast(1) << (64 - count_leading_zeroes64(x)); + return y * Chunk::CHUNK_SIZE; + } + + void shrink() { + size_t want_size = normalize((used_nodes_ + 1) * 5 / 3 + 1); + resize(want_size); + } + + void grow() { + size_t want_size = normalize(2 * nodes_.size() - !nodes_.empty()); + resize(want_size); + } + + struct HashInfo { + size_t chunk_i; + uint8 small_hash; + }; + struct ChunkIt { + size_t chunk_i; + size_t chunk_mask; + size_t shift; + + size_t pos() const { + return chunk_i; + } + void next() { + DCHECK((chunk_mask & (chunk_mask + 1)) == 0); + shift++; + chunk_i += shift; + chunk_i &= chunk_mask; + } + }; + + ChunkIt get_chunk_it(size_t chunk_i) { + return ChunkIt{chunk_i, chunks_.size() - 1, 0}; + } + + HashInfo calc_hash(const KeyT &key) { + auto h = HashT()(key); + return {(h >> 8) % chunks_.size(), static_cast(0x80 | h)}; + } + + void resize(size_t new_size) { + CHECK(new_size >= Chunk::CHUNK_SIZE); + fixed_vector old_nodes(new_size); + fixed_vector chunks(new_size / Chunk::CHUNK_SIZE); + old_nodes.swap(nodes_); + chunks_ = std::move(chunks); + used_nodes_ = 0; + + for (auto &node : old_nodes) { + if (node.empty()) { + continue; + } + emplace_node(std::move(node)); + } + } + + void emplace_node(Node &&node) { + DCHECK(!node.empty()); + auto hash = calc_hash(node.key()); + auto chunk_it = get_chunk_it(hash.chunk_i); + while (true) { + auto chunk_i = chunk_it.pos(); + auto &chunk = chunks_[chunk_i]; + auto mask_it = MaskHelper::equal_mask(chunk.ctrl, 0); + if (mask_it) { + auto shift = mask_it.pos(); + auto node_it = nodes_.begin() + shift + chunk_i * Chunk::CHUNK_SIZE; + DCHECK(node_it->empty()); + *node_it = std::move(node); + DCHECK(chunk.ctrl[shift] == 0); + chunk.ctrl[shift] = hash.small_hash; + DCHECK(chunk.ctrl[shift] != 0); + used_nodes_++; + break; + } + CHECK(chunk.skipped_cnt != std::numeric_limits::max()); + chunk.skipped_cnt++; + chunk_it.next(); + } + } + + void next_bucket(size_t &bucket) const { + bucket++; + if (unlikely(bucket == nodes_.size())) { + bucket = 0; + } + } + + void erase_node(NodeIterator it) { + DCHECK(!it->empty()); + size_t empty_i = it - nodes_.begin(); + DCHECK(0 <= empty_i && empty_i < nodes_.size()); + auto empty_chunk_i = empty_i / Chunk::CHUNK_SIZE; + auto hash = calc_hash(it->key()); + auto chunk_it = get_chunk_it(hash.chunk_i); + while (true) { + auto chunk_i = chunk_it.pos(); + auto &chunk = chunks_[chunk_i]; + if (chunk_i == empty_chunk_i) { + chunk.ctrl[empty_i - empty_chunk_i * Chunk::CHUNK_SIZE] = 0; + break; + } + chunk.skipped_cnt--; + chunk_it.next(); + } + it->clear(); + used_nodes_--; + } +}; + +template , class EqT = std::equal_to> +using FlatHashMapChunks = FlatHashTableChunks, HashT, EqT>; + +template , class EqT = std::equal_to> +using FlatHashSetChunks = FlatHashTableChunks, HashT, EqT>; + +template +void table_remove_if(FlatHashTableChunks &table, FuncT &&func) { + table.remove_if(func); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashSet.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashSet.h new file mode 100644 index 0000000000..385485979a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashSet.h @@ -0,0 +1,24 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +//#include "td/utils/FlatHashMapChunks.h" +#include "td/utils/FlatHashTable.h" +#include "td/utils/HashTableUtils.h" +#include "td/utils/SetNode.h" + +#include +//#include + +namespace td { + +template , class EqT = std::equal_to> +using FlatHashSet = FlatHashTable, HashT, EqT>; +//using FlatHashSet = FlatHashSetChunks; +//using FlatHashSet = std::unordered_set; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.cpp new file mode 100644 index 0000000000..abe7afa764 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.cpp @@ -0,0 +1,24 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/FlatHashTable.h" + +#include "td/utils/bits.h" +#include "td/utils/Random.h" + +namespace td { +namespace detail { + +uint32 normalize_flat_hash_table_size(uint32 size) { + return td::max(static_cast(1) << (32 - count_leading_zeroes32(size)), static_cast(8)); +} + +uint32 get_random_flat_hash_table_bucket(uint32 bucket_count_mask) { + return Random::fast_uint32() & bucket_count_mask; +} + +} // namespace detail +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.h new file mode 100644 index 0000000000..a312ad9533 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FlatHashTable.h @@ -0,0 +1,551 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/HashTableUtils.h" + +#include +#include +#include +#include + +namespace td { + +namespace detail { +uint32 normalize_flat_hash_table_size(uint32 size); +uint32 get_random_flat_hash_table_bucket(uint32 bucket_count_mask); +} // namespace detail + +template +class FlatHashTable { + static constexpr uint32 INVALID_BUCKET = 0xFFFFFFFF; + + void allocate_nodes(uint32 size) { + DCHECK(size >= 8); + DCHECK((size & (size - 1)) == 0); + CHECK(size <= min(static_cast(1) << 29, static_cast(0x7FFFFFFF / sizeof(NodeT)))); + nodes_ = new NodeT[size]; + // used_node_count_ = 0; + bucket_count_mask_ = size - 1; + bucket_count_ = size; + begin_bucket_ = INVALID_BUCKET; + } + + static void clear_nodes(NodeT *nodes) { + delete[] nodes; + } + + public: + using KeyT = typename NodeT::public_key_type; + using key_type = typename NodeT::public_key_type; + using value_type = typename NodeT::public_type; + + // TODO use EndSentinel for end() after switching to C++17 + // struct EndSentinel {}; + + struct Iterator { + using iterator_category = std::forward_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = FlatHashTable::value_type; + using pointer = value_type *; + using reference = value_type &; + + Iterator &operator++() { + DCHECK(it_ != nullptr); + do { + if (unlikely(++it_ == end_)) { + it_ = begin_; + } + if (unlikely(it_ == start_)) { + it_ = nullptr; + break; + } + } while (it_->empty()); + return *this; + } + reference operator*() { + return it_->get_public(); + } + const value_type &operator*() const { + return it_->get_public(); + } + pointer operator->() { + return &it_->get_public(); + } + const value_type *operator->() const { + return &it_->get_public(); + } + + NodeT *get() { + return it_; + } + + bool operator==(const Iterator &other) const { + DCHECK(other.it_ == nullptr); + return it_ == nullptr; + } + bool operator!=(const Iterator &other) const { + DCHECK(other.it_ == nullptr); + return it_ != nullptr; + } + + Iterator() = default; + Iterator(NodeT *it, NodeT *begin, NodeT *end) : it_(it), begin_(begin), start_(it), end_(end) { + } + + private: + NodeT *it_ = nullptr; + NodeT *begin_ = nullptr; + NodeT *start_ = nullptr; + NodeT *end_ = nullptr; + }; + + struct ConstIterator { + using iterator_category = std::forward_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = FlatHashTable::value_type; + using pointer = const value_type *; + using reference = const value_type &; + + ConstIterator &operator++() { + ++it_; + return *this; + } + reference operator*() const { + return *it_; + } + pointer operator->() const { + return &*it_; + } + bool operator==(const ConstIterator &other) const { + return it_ == other.it_; + } + bool operator!=(const ConstIterator &other) const { + return it_ != other.it_; + } + + ConstIterator() = default; + ConstIterator(Iterator it) : it_(std::move(it)) { + } + + private: + Iterator it_; + }; + using iterator = Iterator; + using const_iterator = ConstIterator; + + struct NodePointer { + value_type &operator*() { + return it_->get_public(); + } + const value_type &operator*() const { + return it_->get_public(); + } + value_type *operator->() { + return &it_->get_public(); + } + const value_type *operator->() const { + return &it_->get_public(); + } + + NodeT *get() { + return it_; + } + + bool operator==(const Iterator &) const { + return it_ == nullptr; + } + bool operator!=(const Iterator &) const { + return it_ != nullptr; + } + + explicit NodePointer(NodeT *it) : it_(it) { + } + + private: + NodeT *it_ = nullptr; + }; + + struct ConstNodePointer { + const value_type &operator*() const { + return it_->get_public(); + } + const value_type *operator->() const { + return &it_->get_public(); + } + + bool operator==(const ConstIterator &) const { + return it_ == nullptr; + } + bool operator!=(const ConstIterator &) const { + return it_ != nullptr; + } + + const NodeT *get() const { + return it_; + } + + explicit ConstNodePointer(const NodeT *it) : it_(it) { + } + + private: + const NodeT *it_ = nullptr; + }; + + FlatHashTable() = default; + FlatHashTable(const FlatHashTable &other) = delete; + FlatHashTable &operator=(const FlatHashTable &other) = delete; + + FlatHashTable(std::initializer_list nodes) { + if (nodes.size() == 0) { + return; + } + reserve(nodes.size()); + uint32 used_nodes = 0; + for (auto &new_node : nodes) { + CHECK(!new_node.empty()); + auto bucket = calc_bucket(new_node.key()); + while (true) { + auto &node = nodes_[bucket]; + if (node.empty()) { + node.copy_from(new_node); + used_nodes++; + break; + } + if (EqT()(node.key(), new_node.key())) { + break; + } + next_bucket(bucket); + } + } + used_node_count_ = used_nodes; + } + + FlatHashTable(FlatHashTable &&other) noexcept + : nodes_(other.nodes_) + , used_node_count_(other.used_node_count_) + , bucket_count_mask_(other.bucket_count_mask_) + , bucket_count_(other.bucket_count_) + , begin_bucket_(other.begin_bucket_) { + other.drop(); + } + void operator=(FlatHashTable &&other) noexcept { + clear(); + nodes_ = other.nodes_; + used_node_count_ = other.used_node_count_; + bucket_count_mask_ = other.bucket_count_mask_; + bucket_count_ = other.bucket_count_; + begin_bucket_ = other.begin_bucket_; + other.drop(); + } + ~FlatHashTable() { + clear_nodes(nodes_); + } + + void swap(FlatHashTable &other) noexcept { + std::swap(nodes_, other.nodes_); + std::swap(used_node_count_, other.used_node_count_); + std::swap(bucket_count_mask_, other.bucket_count_mask_); + std::swap(bucket_count_, other.bucket_count_); + std::swap(begin_bucket_, other.begin_bucket_); + } + + uint32 bucket_count() const { + return bucket_count_; + } + + NodePointer find(const KeyT &key) { + return NodePointer(find_impl(key)); + } + + ConstNodePointer find(const KeyT &key) const { + return ConstNodePointer(const_cast(this)->find_impl(key)); + } + + size_t size() const { + return used_node_count_; + } + + bool empty() const { + return used_node_count_ == 0; + } + + Iterator begin() { + return create_iterator(begin_impl()); + } + Iterator end() { + return Iterator(); + } + ConstIterator begin() const { + return ConstIterator(const_cast(this)->begin()); + } + ConstIterator end() const { + return ConstIterator(); + } + + void reserve(size_t size) { + if (size == 0) { + return; + } + CHECK(size <= (1u << 29)); + uint32 want_size = detail::normalize_flat_hash_table_size(static_cast(size) * 5 / 3 + 1); + if (want_size > bucket_count()) { + resize(want_size); + } + } + + template + std::pair emplace(KeyT key, ArgsT &&...args) { + CHECK(!is_hash_table_key_empty(key)); + if (unlikely(bucket_count_mask_ == 0)) { + CHECK(used_node_count_ == 0); + resize(8); + } + auto bucket = calc_bucket(key); + while (true) { + auto &node = nodes_[bucket]; + if (node.empty()) { + if (unlikely(used_node_count_ * 5 >= bucket_count_mask_ * 3)) { + resize(2 * bucket_count_); + CHECK(used_node_count_ * 5 < bucket_count_mask_ * 3); + return emplace(std::move(key), std::forward(args)...); + } + invalidate_iterators(); + + node.emplace(std::move(key), std::forward(args)...); + used_node_count_++; + return {NodePointer(&node), true}; + } + if (EqT()(node.key(), key)) { + return {NodePointer(&node), false}; + } + next_bucket(bucket); + } + } + + std::pair insert(KeyT key) { + return emplace(std::move(key)); + } + + template + void insert(ItT begin, ItT end) { + for (; begin != end; ++begin) { + emplace(*begin); + } + } + + template + T &operator[](const KeyT &key) { + return emplace(key).first->second; + } + + size_t erase(const KeyT &key) { + auto *node = find_impl(key); + if (node == nullptr) { + return 0; + } + erase_node(node); + try_shrink(); + return 1; + } + + size_t count(const KeyT &key) const { + return const_cast(this)->find_impl(key) != nullptr; + } + + void clear() { + if (nodes_ != nullptr) { + clear_nodes(nodes_); + drop(); + } + } + + void erase(Iterator it) { + DCHECK(it != end()); + erase_node(it.get()); + try_shrink(); + } + + void erase(NodePointer it) { + DCHECK(it != end()); + erase_node(it.get()); + try_shrink(); + } + + template + void remove_if(F &&f) { + if (empty()) { + return; + } + + auto it = begin_impl(); + auto end = nodes_ + bucket_count(); + while (it != end && !it->empty()) { + ++it; + } + if (it == end) { + do { + --it; + } while (!it->empty()); + } + auto first_empty = it; + while (it != end) { + if (!it->empty() && f(it->get_public())) { + erase_node(it); + } else { + ++it; + } + } + for (it = nodes_; it != first_empty;) { + if (!it->empty() && f(it->get_public())) { + erase_node(it); + } else { + ++it; + } + } + try_shrink(); + } + + private: + NodeT *nodes_ = nullptr; + uint32 used_node_count_ = 0; + uint32 bucket_count_mask_ = 0; + uint32 bucket_count_ = 0; + uint32 begin_bucket_ = 0; + + void drop() { + nodes_ = nullptr; + used_node_count_ = 0; + bucket_count_mask_ = 0; + bucket_count_ = 0; + begin_bucket_ = 0; + } + + NodeT *begin_impl() { + if (empty()) { + return nullptr; + } + if (begin_bucket_ == INVALID_BUCKET) { + begin_bucket_ = detail::get_random_flat_hash_table_bucket(bucket_count_mask_); + while (nodes_[begin_bucket_].empty()) { + next_bucket(begin_bucket_); + } + } + return nodes_ + begin_bucket_; + } + + NodeT *find_impl(const KeyT &key) { + if (unlikely(nodes_ == nullptr) || is_hash_table_key_empty(key)) { + return nullptr; + } + auto bucket = calc_bucket(key); + while (true) { + auto &node = nodes_[bucket]; + if (node.empty()) { + return nullptr; + } + if (EqT()(node.key(), key)) { + return &node; + } + next_bucket(bucket); + } + } + + void try_shrink() { + DCHECK(nodes_ != nullptr); + if (unlikely(used_node_count_ * 10 < bucket_count_mask_ && bucket_count_mask_ > 7)) { + resize(detail::normalize_flat_hash_table_size((used_node_count_ + 1) * 5 / 3 + 1)); + } + invalidate_iterators(); + } + + uint32 calc_bucket(const KeyT &key) const { + return HashT()(key) & bucket_count_mask_; + } + + inline void next_bucket(uint32 &bucket) const { + bucket = (bucket + 1) & bucket_count_mask_; + } + + void resize(uint32 new_size) { + if (unlikely(nodes_ == nullptr)) { + allocate_nodes(new_size); + used_node_count_ = 0; + return; + } + + auto old_nodes = nodes_; + uint32 old_size = used_node_count_; + uint32 old_bucket_count = bucket_count_; + allocate_nodes(new_size); + used_node_count_ = old_size; + + auto old_nodes_end = old_nodes + old_bucket_count; + for (NodeT *old_node = old_nodes; old_node != old_nodes_end; ++old_node) { + if (old_node->empty()) { + continue; + } + auto bucket = calc_bucket(old_node->key()); + while (!nodes_[bucket].empty()) { + next_bucket(bucket); + } + nodes_[bucket] = std::move(*old_node); + } + clear_nodes(old_nodes); + } + + void erase_node(NodeT *it) { + DCHECK(nodes_ <= it && static_cast(it - nodes_) < bucket_count()); + it->clear(); + used_node_count_--; + + const auto bucket_count = bucket_count_; + const auto *end = nodes_ + bucket_count; + for (auto *test_node = it + 1; test_node != end; test_node++) { + if (likely(test_node->empty())) { + return; + } + + auto want_node = nodes_ + calc_bucket(test_node->key()); + if (want_node <= it || want_node > test_node) { + *it = std::move(*test_node); + it = test_node; + } + } + + auto empty_i = static_cast(it - nodes_); + auto empty_bucket = empty_i; + for (uint32 test_i = bucket_count;; test_i++) { + auto test_bucket = test_i - bucket_count_; + if (nodes_[test_bucket].empty()) { + return; + } + + auto want_i = calc_bucket(nodes_[test_bucket].key()); + if (want_i < empty_i) { + want_i += bucket_count; + } + + if (want_i <= empty_i || want_i > test_i) { + nodes_[empty_bucket] = std::move(nodes_[test_bucket]); + empty_i = test_i; + empty_bucket = test_bucket; + } + } + } + + Iterator create_iterator(NodeT *node) { + return Iterator(node, nodes_, nodes_ + bucket_count()); + } + + void invalidate_iterators() { + begin_bucket_ = INVALID_BUCKET; + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h index 9f047881aa..e5570f6d2a 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,56 +7,81 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/TimedStat.h" namespace td { class FloodControlFast { public: - uint32 add_event(int32 now) { - for (auto &limit : limits_) { - limit.stat_.add_event(CounterStat::Event(), now); - if (limit.stat_.get_stat(now).count_ > limit.count_) { - wakeup_at_ = max(wakeup_at_, now + limit.duration_ * 2); - } + void add_event(double now) { + for (auto &bucket : buckets_) { + bucket.add_event(now); + wakeup_at_ = td::max(wakeup_at_, bucket.get_wakeup_at()); } - return wakeup_at_; } - uint32 get_wakeup_at() { + + double get_wakeup_at() const { return wakeup_at_; } - void add_limit(uint32 duration, int32 count) { - limits_.push_back({TimedStat(duration, 0), duration, count}); + void add_limit(double duration, double count) { + buckets_.emplace_back(duration, count); } void clear_events() { - for (auto &limit : limits_) { - limit.stat_.clear_events(); + for (auto &bucket : buckets_) { + bucket.clear_events(); } wakeup_at_ = 0; } private: - class CounterStat { + class FloodControlBucket { public: - struct Event {}; - int32 count_ = 0; - void on_event(Event e) { - count_++; + FloodControlBucket(double duration, double count) + : max_capacity_(count - 1), speed_(count / duration), volume_(max_capacity_) { } - void clear() { - count_ = 0; + + void add_event(double now, double size = 1) { + CHECK(now >= wakeup_at_); + update_volume(now); + if (volume_ >= size) { + volume_ -= size; + return; + } + size -= volume_; + volume_ = 0; + wakeup_at_ = volume_at_ + size / speed_; + volume_at_ = wakeup_at_; } - }; - uint32 wakeup_at_ = 0; - struct Limit { - TimedStat stat_; - uint32 duration_; - int32 count_; + double get_wakeup_at() const { + return wakeup_at_; + } + + void clear_events() { + volume_ = max_capacity_; + volume_at_ = 0; + wakeup_at_ = 0; + } + + private: + const double max_capacity_{1}; + const double speed_{1}; + double volume_{1}; + + double volume_at_{0}; + double wakeup_at_{0}; + + void update_volume(double now) { + CHECK(now >= volume_at_); + auto passed = now - volume_at_; + volume_ = td::min(volume_ + passed * speed_, max_capacity_); + volume_at_ = now; + } }; - std::vector limits_; + + double wakeup_at_ = 0; + vector buckets_; }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.cpp new file mode 100644 index 0000000000..98709c6b96 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.cpp @@ -0,0 +1,32 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/FloodControlGlobal.h" + +namespace td { + +FloodControlGlobal::FloodControlGlobal(uint64 limit) : limit_(limit) { +} + +void FloodControlGlobal::finish() { + auto old_value = active_count_.fetch_sub(1, std::memory_order_relaxed); + CHECK(old_value > 0); +} + +FloodControlGlobal::Guard FloodControlGlobal::try_start() { + auto old_value = active_count_.fetch_add(1, std::memory_order_relaxed); + if (old_value >= limit_) { + finish(); + return nullptr; + } + return Guard(this); +} + +void FloodControlGlobal::Finish::operator()(FloodControlGlobal *ctrl) const { + ctrl->finish(); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.h new file mode 100644 index 0000000000..e4f318867b --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlGlobal.h @@ -0,0 +1,35 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include +#include + +namespace td { + +// Restricts the total number of events +class FloodControlGlobal { + public: + explicit FloodControlGlobal(uint64 limit); + + struct Finish { + void operator()(FloodControlGlobal *ctrl) const; + }; + using Guard = std::unique_ptr; + + Guard try_start(); + + private: + std::atomic active_count_{0}; + uint64 limit_{0}; + + void finish(); +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h index 521fbbedc0..42894cdf7f 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,7 +7,6 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/logging.h" #include @@ -17,22 +16,23 @@ namespace td { // Should be just fine for small counters. class FloodControlStrict { public: - int32 add_event(int32 now) { + // there is no reason to return wakeup_at_, because it will be a time before the next allowed event, not current + void add_event(double now) { events_.push_back(Event{now}); if (without_update_ > 0) { without_update_--; } else { update(now); } - return wakeup_at_; } - // no more than count in each duration. - void add_limit(int32 duration, int32 count) { + // no more than count in each duration + void add_limit(int32 duration, size_t count) { limits_.push_back(Limit{duration, count, 0}); + without_update_ = 0; } - int32 get_wakeup_at() { + double get_wakeup_at() const { return wakeup_at_; } @@ -42,20 +42,22 @@ class FloodControlStrict { limit.pos_ = 0; } without_update_ = 0; - wakeup_at_ = 0; + wakeup_at_ = 0.0; } - int32 update(int32 now) { + private: + void update(double now) { size_t min_pos = events_.size(); without_update_ = std::numeric_limits::max(); for (auto &limit : limits_) { - if (limit.pos_ + limit.count_ < events_.size()) { + if (limit.count_ < events_.size() - limit.pos_) { limit.pos_ = events_.size() - limit.count_; } // binary-search? :D - while (limit.pos_ < events_.size() && events_[limit.pos_].timestamp_ + limit.duration_ < now) { + auto end_time = now - limit.duration_; + while (limit.pos_ < events_.size() && events_[limit.pos_].timestamp_ < end_time) { limit.pos_++; } @@ -64,7 +66,7 @@ class FloodControlStrict { wakeup_at_ = max(wakeup_at_, events_[limit.pos_].timestamp_ + limit.duration_); without_update_ = 0; } else { - without_update_ = min(without_update_, limit.count_ + limit.pos_ - events_.size()); + without_update_ = min(without_update_, limit.count_ + limit.pos_ - events_.size() - 1); } min_pos = min(min_pos, limit.pos_); @@ -76,22 +78,20 @@ class FloodControlStrict { } events_.erase(events_.begin(), events_.begin() + min_pos); } - return wakeup_at_; } - private: - int32 wakeup_at_ = 0; + double wakeup_at_ = 0.0; struct Event { - int32 timestamp_; + double timestamp_; }; struct Limit { int32 duration_; - int32 count_; + size_t count_; size_t pos_; }; size_t without_update_ = 0; - std::vector events_; - std::vector limits_; + vector events_; + vector limits_; }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp deleted file mode 100644 index 976286b923..0000000000 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -#include "td/utils/GitInfo.h" - -#include "auto/git_info.h" - -namespace td { - -CSlice GitInfo::commit() { - return GIT_COMMIT; -} -bool GitInfo::is_dirty() { - return GIT_DIRTY; -} - -} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h deleted file mode 100644 index a3ba32602f..0000000000 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h +++ /dev/null @@ -1,19 +0,0 @@ -// -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -#pragma once - -#include "td/utils/Slice.h" - -namespace td { - -class GitInfo { - public: - static CSlice commit(); - static bool is_dirty(); -}; - -} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp index d4e60d6e29..64b07a9c04 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -9,10 +9,11 @@ char disable_linker_warning_about_empty_file_gzip_cpp TD_UNUSED; #if TD_HAVE_ZLIB -#include "td/utils/logging.h" +#include "td/utils/SliceBuilder.h" #include #include +#include #include @@ -32,23 +33,23 @@ class Gzip::Impl { }; Status Gzip::init_encode() { - CHECK(mode_ == Empty); + CHECK(mode_ == Mode::Empty); init_common(); - mode_ = Encode; + mode_ = Mode::Encode; int ret = deflateInit2(&impl_->stream_, 6, Z_DEFLATED, 15, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY); if (ret != Z_OK) { - return Status::Error("zlib deflate init failed"); + return Status::Error(PSLICE() << "zlib deflate init failed: " << ret); } return Status::OK(); } Status Gzip::init_decode() { - CHECK(mode_ == Empty); + CHECK(mode_ == Mode::Empty); init_common(); - mode_ = Decode; + mode_ = Mode::Decode; int ret = inflateInit2(&impl_->stream_, MAX_WBITS + 32); if (ret != Z_OK) { - return Status::Error("zlib inflate init failed"); + return Status::Error(PSLICE() << "zlib inflate init failed: " << ret); } return Status::OK(); } @@ -75,19 +76,19 @@ void Gzip::set_output(MutableSlice output) { Result Gzip::run() { while (true) { int ret; - if (mode_ == Decode) { + if (mode_ == Mode::Decode) { ret = inflate(&impl_->stream_, Z_NO_FLUSH); } else { ret = deflate(&impl_->stream_, close_input_flag_ ? Z_FINISH : Z_NO_FLUSH); } if (ret == Z_OK) { - return Running; + return State::Running; } if (ret == Z_STREAM_END) { // TODO(now): fail if input is not empty; clear(); - return Done; + return State::Done; } clear(); return Status::Error(PSLICE() << "zlib error " << ret); @@ -118,20 +119,36 @@ void Gzip::init_common() { } void Gzip::clear() { - if (mode_ == Decode) { + if (mode_ == Mode::Decode) { inflateEnd(&impl_->stream_); - } else if (mode_ == Encode) { + } else if (mode_ == Mode::Encode) { deflateEnd(&impl_->stream_); } - mode_ = Empty; + mode_ = Mode::Empty; } Gzip::Gzip() : impl_(make_unique()) { } -Gzip::Gzip(Gzip &&other) = default; +Gzip::Gzip(Gzip &&other) noexcept : Gzip() { + swap(other); +} -Gzip &Gzip::operator=(Gzip &&other) = default; +Gzip &Gzip::operator=(Gzip &&other) noexcept { + CHECK(this != &other); + clear(); + swap(other); + return *this; +} + +void Gzip::swap(Gzip &other) { + using std::swap; + swap(impl_, other.impl_); + swap(input_size_, other.input_size_); + swap(output_size_, other.output_size_); + swap(close_input_flag_, other.close_input_flag_); + swap(mode_, other.mode_); +} Gzip::~Gzip() { clear(); @@ -140,7 +157,7 @@ Gzip::~Gzip() { BufferSlice gzdecode(Slice s) { Gzip gzip; gzip.init_decode().ensure(); - auto message = ChainBufferWriter::create_empty(); + ChainBufferWriter message; gzip.set_input(s); gzip.close_input(); double k = 2; @@ -151,7 +168,7 @@ BufferSlice gzdecode(Slice s) { return BufferSlice(); } auto state = r_state.ok(); - if (state == Gzip::Done) { + if (state == Gzip::State::Done) { message.confirm_append(gzip.flush_output()); break; } @@ -167,12 +184,12 @@ BufferSlice gzdecode(Slice s) { return message.extract_reader().move_as_buffer_slice(); } -BufferSlice gzencode(Slice s, double k) { +BufferSlice gzencode(Slice s, double max_compression_ratio) { Gzip gzip; gzip.init_encode().ensure(); gzip.set_input(s); gzip.close_input(); - size_t max_size = static_cast(static_cast(s.size()) * k); + auto max_size = static_cast(static_cast(s.size()) * max_compression_ratio); BufferWriter message{max_size}; gzip.set_output(message.prepare_append()); auto r_state = gzip.run(); @@ -180,7 +197,7 @@ BufferSlice gzencode(Slice s, double k) { return BufferSlice(); } auto state = r_state.ok(); - if (state != Gzip::Done) { + if (state != Gzip::State::Done) { return BufferSlice(); } message.confirm_append(gzip.flush_output()); diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h index dd5fba5bf5..d7b68c5e45 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -20,15 +20,15 @@ class Gzip { Gzip(); Gzip(const Gzip &) = delete; Gzip &operator=(const Gzip &) = delete; - Gzip(Gzip &&other); - Gzip &operator=(Gzip &&other); + Gzip(Gzip &&other) noexcept; + Gzip &operator=(Gzip &&other) noexcept; ~Gzip(); - enum Mode { Empty, Encode, Decode }; + enum class Mode { Empty, Encode, Decode }; Status init(Mode mode) TD_WARN_UNUSED_RESULT { - if (mode == Encode) { + if (mode == Mode::Encode) { return init_encode(); - } else if (mode == Decode) { + } else if (mode == Mode::Decode) { return init_decode(); } clear(); @@ -79,7 +79,7 @@ class Gzip { return res; } - enum State { Running, Done }; + enum class State { Running, Done }; Result run() TD_WARN_UNUSED_RESULT; private: @@ -89,15 +89,17 @@ class Gzip { size_t input_size_ = 0; size_t output_size_ = 0; bool close_input_flag_ = false; - Mode mode_ = Empty; + Mode mode_ = Mode::Empty; void init_common(); void clear(); + + void swap(Gzip &other); }; BufferSlice gzdecode(Slice s); -BufferSlice gzencode(Slice s, double k = 0.9); +BufferSlice gzencode(Slice s, double max_compression_ratio); } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp index d225ef800e..d321b68ccf 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -9,62 +9,57 @@ char disable_linker_warning_about_empty_file_gzipbyteflow_cpp TD_UNUSED; #if TD_HAVE_ZLIB -#include "td/utils/logging.h" +#include "td/utils/common.h" #include "td/utils/Status.h" namespace td { -void GzipByteFlow::loop() { - while (true) { - if (gzip_.need_input()) { - auto slice = input_->prepare_read(); - if (slice.empty()) { - if (!is_input_active_) { - gzip_.close_input(); - } else { - break; - } +bool GzipByteFlow::loop() { + if (gzip_.need_input()) { + auto slice = input_->prepare_read(); + if (slice.empty()) { + if (!is_input_active_) { + gzip_.close_input(); } else { - gzip_.set_input(input_->prepare_read()); + return false; } + } else { + gzip_.set_input(input_->prepare_read()); } - if (gzip_.need_output()) { - auto slice = output_.prepare_append(); - CHECK(!slice.empty()); - gzip_.set_output(slice); - } - auto r_state = gzip_.run(); - auto output_size = gzip_.flush_output(); - if (output_size) { - uncommited_size_ += output_size; - total_output_size_ += output_size; - if (total_output_size_ > max_output_size_) { - return finish(Status::Error("Max output size limit exceeded")); - } - output_.confirm_append(output_size); + } + if (gzip_.need_output()) { + auto slice = output_.prepare_append(); + CHECK(!slice.empty()); + gzip_.set_output(slice); + } + auto r_state = gzip_.run(); + auto output_size = gzip_.flush_output(); + if (output_size) { + uncommitted_size_ += output_size; + total_output_size_ += output_size; + if (total_output_size_ > max_output_size_) { + finish(Status::Error("Max output size limit exceeded")); + return false; } + output_.confirm_append(output_size); + } - auto input_size = gzip_.flush_input(); - if (input_size) { - input_->confirm_read(input_size); - } - if (r_state.is_error()) { - return finish(r_state.move_as_error()); - } - auto state = r_state.ok(); - if (state == Gzip::Done) { - on_output_updated(); - return consume_input(); - } + auto input_size = gzip_.flush_input(); + if (input_size) { + input_->confirm_read(input_size); } - if (uncommited_size_ >= MIN_UPDATE_SIZE) { - uncommited_size_ = 0; - on_output_updated(); + if (r_state.is_error()) { + finish(r_state.move_as_error()); + return false; } + auto state = r_state.ok(); + if (state == Gzip::State::Done) { + consume_input(); + return false; + } + return true; } -constexpr size_t GzipByteFlow::MIN_UPDATE_SIZE; - } // namespace td #endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h index c7e07abd0a..94a3a3ea50 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -34,14 +34,13 @@ class GzipByteFlow final : public ByteFlowBase { max_output_size_ = max_output_size; } - void loop() override; + bool loop() final; private: Gzip gzip_; - size_t uncommited_size_ = 0; + size_t uncommitted_size_ = 0; size_t total_output_size_ = 0; size_t max_output_size_ = std::numeric_limits::max(); - static constexpr size_t MIN_UPDATE_SIZE = 1 << 14; }; #endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hash.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hash.h new file mode 100644 index 0000000000..8c500daf09 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hash.h @@ -0,0 +1,70 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#if TD_HAVE_ABSL +#include +#endif + +#include + +namespace td { +// A simple wrapper for absl::flat_hash_map, std::unordered_map and probably some our implementaion of hash map in +// the future + +// We will introduce out own Hashing utility like an absl one. +class Hasher { + public: + Hasher() = default; + explicit Hasher(size_t init_value) : hash_(init_value) { + } + std::size_t finalize() const { + return hash_; + } + + static Hasher combine(Hasher hasher, size_t value) { + hasher.hash_ ^= value; + return hasher; + } + + template + static Hasher combine(Hasher hasher, const std::pair &value) { + hasher = AbslHashValue(std::move(hasher), value.first); + hasher = AbslHashValue(std::move(hasher), value.second); + return hasher; + } + + private: + std::size_t hash_{0}; +}; + +template +class TdHash { + public: + template + std::size_t operator()(const T &value) const noexcept { + return AbslHashValue(Hasher(), value).finalize(); + } +}; + +#if TD_HAVE_ABSL +template +using AbslHash = absl::Hash; +#else +template +using AbslHash = TdHash; +#endif + +// default hash implementations +template +decltype(H::combine(std::declval(), std::declval())) AbslHashValue(H hasher, const T &value) { + return H::combine(std::move(hasher), value); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HashMap.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HashMap.h new file mode 100644 index 0000000000..7e0ba4bf07 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HashMap.h @@ -0,0 +1,27 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Hash.h" + +#if TD_HAVE_ABSL +#include +#else +#include +#endif + +namespace td { + +#if TD_HAVE_ABSL +template > +using HashMap = absl::flat_hash_map; +#else +template > +using HashMap = std::unordered_map; +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HashSet.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HashSet.h new file mode 100644 index 0000000000..e49e8b94e3 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HashSet.h @@ -0,0 +1,27 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Hash.h" + +#if TD_HAVE_ABSL +#include +#else +#include +#endif + +namespace td { + +#if TD_HAVE_ABSL +template > +using HashSet = absl::flat_hash_set; +#else +template > +using HashSet = std::unordered_set; +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HashTableUtils.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HashTableUtils.h new file mode 100644 index 0000000000..9d72f63c59 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HashTableUtils.h @@ -0,0 +1,72 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include +#include + +namespace td { + +template +bool is_hash_table_key_empty(const KeyT &key) { + return key == KeyT(); +} + +inline uint32 randomize_hash(uint32 h) { + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +template +struct Hash { + uint32 operator()(const Type &value) const; +}; + +template +struct Hash { + uint32 operator()(Type *pointer) const { + return Hash()(reinterpret_cast(pointer)); + } +}; + +template <> +inline uint32 Hash::operator()(const char &value) const { + return randomize_hash(static_cast(value)); +} + +template <> +inline uint32 Hash::operator()(const int32 &value) const { + return randomize_hash(static_cast(value)); +} + +template <> +inline uint32 Hash::operator()(const uint32 &value) const { + return randomize_hash(value); +} + +template <> +inline uint32 Hash::operator()(const int64 &value) const { + return randomize_hash(static_cast(value + (value >> 32))); +} + +template <> +inline uint32 Hash::operator()(const uint64 &value) const { + return randomize_hash(static_cast(value + (value >> 32))); +} + +template <> +inline uint32 Hash::operator()(const string &value) const { + return static_cast(std::hash()(value)); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h index e13dc8022e..3ed41a0c9a 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,20 +7,25 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/logging.h" #include #include +#include namespace td { -template +template > class HazardPointers { public: explicit HazardPointers(size_t threads_n) : threads_(threads_n) { for (auto &data : threads_) { - for (auto &ptr : data.hazard) { + for (auto &ptr : data.hazard_) { +// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658 +#if TD_GCC && GCC_VERSION <= 40902 ptr = nullptr; +#else + std::atomic_init(&ptr, static_cast(nullptr)); +#endif } } } @@ -31,12 +36,17 @@ class HazardPointers { class Holder { public: - T *protect(std::atomic &to_protect) { + template + S *protect(std::atomic &to_protect) { return do_protect(hazard_ptr_, to_protect); } + Holder(HazardPointers &hp, size_t thread_id, size_t pos) : Holder(hp.get_hazard_ptr(thread_id, pos)) { + CHECK(hazard_ptr_.load() == 0); + hazard_ptr_.store(reinterpret_cast(1)); + } Holder(const Holder &other) = delete; Holder &operator=(const Holder &other) = delete; - Holder(Holder &&other) = default; // TODO + Holder(Holder &&other) = delete; Holder &operator=(Holder &&other) = delete; ~Holder() { clear(); @@ -52,20 +62,16 @@ class HazardPointers { std::atomic &hazard_ptr_; }; - Holder get_holder(size_t thread_id, size_t pos) { - return Holder(get_hazard_ptr(thread_id, pos)); - } - void retire(size_t thread_id, T *ptr = nullptr) { CHECK(thread_id < threads_.size()); auto &data = threads_[thread_id]; if (ptr) { - data.to_delete.push_back(std::unique_ptr(ptr)); + data.to_delete_.push_back(std::unique_ptr(ptr)); } - for (auto it = data.to_delete.begin(); it != data.to_delete.end();) { + for (auto it = data.to_delete_.begin(); it != data.to_delete_.end();) { if (!is_protected(it->get())) { it->reset(); - it = data.to_delete.erase(it); + it = data.to_delete_.erase(it); } else { ++it; } @@ -82,32 +88,33 @@ class HazardPointers { size_t to_delete_size_unsafe() const { size_t res = 0; - for (auto &thread : threads_) { - res += thread.to_delete.size(); + for (auto &thread_data : threads_) { + res += thread_data.to_delete_.size(); } return res; } private: struct ThreadData { - std::array, MaxPointersN> hazard; + std::array, MaxPointersN> hazard_; char pad[TD_CONCURRENCY_PAD - sizeof(std::array, MaxPointersN>)]; // stupid gc - std::vector> to_delete; - char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector>)]; + std::vector> to_delete_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector>)]; }; std::vector threads_; char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector)]; - static T *do_protect(std::atomic &hazard_ptr, std::atomic &to_protect) { + template + static S *do_protect(std::atomic &hazard_ptr, std::atomic &to_protect) { T *saved = nullptr; T *to_save; while ((to_save = to_protect.load()) != saved) { hazard_ptr.store(to_save); saved = to_save; } - return saved; + return static_cast(saved); } static void do_clear(std::atomic &hazard_ptr) { @@ -115,8 +122,8 @@ class HazardPointers { } bool is_protected(T *ptr) { - for (auto &thread : threads_) { - for (auto &hazard_ptr : thread.hazard) { + for (auto &thread_data : threads_) { + for (auto &hazard_ptr : thread_data.hazard_) { if (hazard_ptr.load() == ptr) { return true; } @@ -126,7 +133,8 @@ class HazardPointers { } std::atomic &get_hazard_ptr(size_t thread_id, size_t pos) { - return threads_[thread_id].hazard[pos]; + CHECK(thread_id < threads_.size()); + return threads_[thread_id].hazard_[pos]; } }; diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h index 54ee391497..154b87089a 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,7 +7,6 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/logging.h" namespace td { @@ -21,7 +20,7 @@ struct HeapNode { void remove() { pos_ = -1; } - int pos_ = -1; + int32 pos_ = -1; }; template @@ -38,23 +37,33 @@ class KHeap { return array_[0].key_; } + KeyT get_key(const HeapNode *node) const { + auto pos = static_cast(node->pos_); + CHECK(pos < array_.size()); + return array_[pos].key_; + } + + const HeapNode *top() const { + return array_[0].node_; + } + HeapNode *pop() { CHECK(!empty()); HeapNode *result = array_[0].node_; result->remove(); - erase(0); + erase(static_cast(0)); return result; } void insert(KeyT key, HeapNode *node) { CHECK(!node->in_heap()); array_.push_back({key, node}); - fix_up(static_cast(array_.size()) - 1); + fix_up(array_.size() - 1); } void fix(KeyT key, HeapNode *node) { - CHECK(node->in_heap()); - int pos = node->pos_; + auto pos = static_cast(node->pos_); + CHECK(pos < array_.size()); KeyT old_key = array_[pos].key_; array_[pos].key_ = key; if (key < old_key) { @@ -65,14 +74,21 @@ class KHeap { } void erase(HeapNode *node) { - CHECK(node->in_heap()); - int pos = node->pos_; + auto pos = static_cast(node->pos_); node->remove(); + CHECK(pos < array_.size()); erase(pos); } template - void for_each(F &f) const { + void for_each(F &&f) const { + for (auto &it : array_) { + f(it.key_, it.node_); + } + } + + template + void for_each(F &&f) { for (auto &it : array_) { f(it.key_, it.node_); } @@ -81,7 +97,7 @@ class KHeap { void check() const { for (size_t i = 0; i < array_.size(); i++) { for (size_t j = i * K + 1; j < i * K + 1 + K && j < array_.size(); j++) { - CHECK(array_[i].key_ <= array_[j].key_) << i << " " << j; + CHECK(array_[i].key_ <= array_[j].key_); } } } @@ -93,34 +109,34 @@ class KHeap { }; vector array_; - void fix_up(int pos) { + void fix_up(size_t pos) { auto item = array_[pos]; while (pos) { - int parent_pos = (pos - 1) / K; + auto parent_pos = (pos - 1) / K; auto parent_item = array_[parent_pos]; if (parent_item.key_ < item.key_) { break; } - parent_item.node_->pos_ = pos; + parent_item.node_->pos_ = static_cast(pos); array_[pos] = parent_item; pos = parent_pos; } - item.node_->pos_ = pos; + item.node_->pos_ = static_cast(pos); array_[pos] = item; } - void fix_down(int pos) { + void fix_down(size_t pos) { auto item = array_[pos]; while (true) { - int left_pos = pos * K + 1; - int right_pos = min(left_pos + K, static_cast(array_.size())); - int next_pos = pos; + auto left_pos = pos * K + 1; + auto right_pos = min(left_pos + K, array_.size()); + auto next_pos = pos; KeyT next_key = item.key_; - for (int i = left_pos; i < right_pos; i++) { + for (auto i = left_pos; i < right_pos; i++) { KeyT i_key = array_[i].key_; if (i_key < next_key) { next_key = i_key; @@ -131,21 +147,24 @@ class KHeap { break; } array_[pos] = array_[next_pos]; - array_[pos].node_->pos_ = pos; + array_[pos].node_->pos_ = static_cast(pos); pos = next_pos; } - item.node_->pos_ = pos; + item.node_->pos_ = static_cast(pos); array_[pos] = item; } - void erase(int pos) { + void erase(size_t pos) { array_[pos] = array_.back(); array_.pop_back(); - if (pos < static_cast(array_.size())) { + if (pos < array_.size()) { fix_down(pos); fix_up(pos); } + if (array_.capacity() > 50 && array_.size() < array_.capacity() / 4) { + array_.shrink_to_fit(); + } } }; diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp index 1e7449a668..1041b95d8b 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp @@ -1,49 +1,23 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include "td/utils/Hints.h" +#include "td/utils/algorithm.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/Slice.h" -#include "td/utils/unicode.h" +#include "td/utils/translit.h" #include "td/utils/utf8.h" #include namespace td { -vector Hints::get_words(Slice name) { - bool in_word = false; - string word; - vector words; - auto pos = name.ubegin(); - auto end = name.uend(); - while (pos != end) { - uint32 code; - pos = next_utf8_unsafe(pos, &code); - - code = prepare_search_character(code); - if (code == 0) { - continue; - } - if (code == ' ') { - if (in_word) { - words.push_back(std::move(word)); - word.clear(); - in_word = false; - } - } else { - in_word = true; - append_utf8_character(word, code); - } - } - if (in_word) { - words.push_back(std::move(word)); - } +vector Hints::fix_words(vector words) { std::sort(words.begin(), words.end()); size_t new_words_size = 0; @@ -52,14 +26,39 @@ vector Hints::get_words(Slice name) { if (i != new_words_size) { words[new_words_size] = std::move(words[i]); } - // LOG(ERROR) << "Get word " << words[new_words_size]; new_words_size++; } } + if (new_words_size == 1 && words[0].empty()) { + new_words_size = 0; + } words.resize(new_words_size); return words; } +vector Hints::get_words(Slice name) { + return fix_words(utf8_get_search_words(name)); +} + +void Hints::add_word(const string &word, KeyT key, std::map> &word_to_keys) { + vector &keys = word_to_keys[word]; + CHECK(!td::contains(keys, key)); + keys.push_back(key); +} + +void Hints::delete_word(const string &word, KeyT key, std::map> &word_to_keys) { + vector &keys = word_to_keys[word]; + auto key_it = std::find(keys.begin(), keys.end(), key); + CHECK(key_it != keys.end()); + if (keys.size() == 1) { + word_to_keys.erase(word); + } else { + CHECK(keys.size() > 1); + *key_it = keys.back(); + keys.pop_back(); + } +} + void Hints::add(KeyT key, Slice name) { // LOG(ERROR) << "Add " << key << ": " << name; auto it = key_to_name_.find(key); @@ -67,19 +66,19 @@ void Hints::add(KeyT key, Slice name) { if (it->second == name) { return; } - auto old_words = get_words(it->second); - for (auto &old_word : old_words) { - vector &keys = word_to_keys_[old_word]; - auto key_it = std::find(keys.begin(), keys.end(), key); - CHECK(key_it != keys.end()); - if (keys.size() == 1) { - word_to_keys_.erase(old_word); - } else { - CHECK(keys.size() > 1); - *key_it = keys.back(); - keys.pop_back(); + vector old_transliterations; + for (auto &old_word : get_words(it->second)) { + delete_word(old_word, key, word_to_keys_); + + for (auto &w : get_word_transliterations(old_word, false)) { + if (w != old_word) { + old_transliterations.push_back(std::move(w)); + } } } + for (auto &word : fix_words(old_transliterations)) { + delete_word(word, key, translit_word_to_keys_); + } } if (name.empty()) { if (it != key_to_name_.end()) { @@ -88,12 +87,21 @@ void Hints::add(KeyT key, Slice name) { key_to_rating_.erase(key); return; } - auto words = get_words(name); - for (auto &word : words) { - vector &keys = word_to_keys_[word]; - CHECK(std::find(keys.begin(), keys.end(), key) == keys.end()); - keys.push_back(key); + + vector transliterations; + for (auto &word : get_words(name)) { + add_word(word, key, word_to_keys_); + + for (auto &w : get_word_transliterations(word, false)) { + if (w != word) { + transliterations.push_back(std::move(w)); + } + } + } + for (auto &word : fix_words(transliterations)) { + add_word(word, key, translit_word_to_keys_); } + key_to_name_[key] = name.str(); } @@ -102,17 +110,24 @@ void Hints::set_rating(KeyT key, RatingT rating) { key_to_rating_[key] = rating; } -vector Hints::search_word(const string &word) const { - // LOG(ERROR) << "Search word " << word; - vector results; - auto it = word_to_keys_.lower_bound(word); - while (it != word_to_keys_.end() && begins_with(it->first, word)) { +void Hints::add_search_results(vector &results, const string &word, + const std::map> &word_to_keys) { + LOG(DEBUG) << "Search for word " << word; + auto it = word_to_keys.lower_bound(word); + while (it != word_to_keys.end() && begins_with(it->first, word)) { results.insert(results.end(), it->second.begin(), it->second.end()); ++it; } +} + +vector Hints::search_word(const string &word) const { + vector results; + add_search_results(results, word, translit_word_to_keys_); + for (const auto &w : get_word_transliterations(word, true)) { + add_search_results(results, w, word_to_keys_); + } - std::sort(results.begin(), results.end()); - results.erase(std::unique(results.begin(), results.end()), results.end()); + td::unique(results); return results; } @@ -169,7 +184,7 @@ std::pair> Hints::search(Slice query, int32 limit, b } bool Hints::has_key(KeyT key) const { - return key_to_name_.find(key) != key_to_name_.end(); + return key_to_name_.count(key) > 0; } string Hints::key_to_string(KeyT key) const { diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h index 645896684a..f069da20d0 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,6 +7,7 @@ #pragma once #include "td/utils/common.h" +#include "td/utils/HashTableUtils.h" #include "td/utils/Slice.h" #include @@ -41,17 +42,26 @@ class Hints { size_t size() const; + static vector fix_words(vector words); + private: std::map> word_to_keys_; - std::unordered_map key_to_name_; - std::unordered_map key_to_rating_; + std::map> translit_word_to_keys_; + std::unordered_map> key_to_name_; + std::unordered_map> key_to_rating_; + + static void add_word(const string &word, KeyT key, std::map> &word_to_keys); + static void delete_word(const string &word, KeyT key, std::map> &word_to_keys); static vector get_words(Slice name); + static void add_search_results(vector &results, const string &word, + const std::map> &word_to_keys); + vector search_word(const string &word) const; class CompareByRating { - const std::unordered_map &key_to_rating_; + const std::unordered_map> &key_to_rating_; RatingT get_rating(const KeyT &key) const { auto it = key_to_rating_.find(key); @@ -62,7 +72,8 @@ class Hints { } public: - explicit CompareByRating(const std::unordered_map &key_to_rating) : key_to_rating_(key_to_rating) { + explicit CompareByRating(const std::unordered_map> &key_to_rating) + : key_to_rating_(key_to_rating) { } bool operator()(const KeyT &lhs, const KeyT &rhs) const { diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp index 55b66f7b3a..e793f940a8 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -10,16 +10,19 @@ #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/Parser.h" +#include "td/utils/port/IPAddress.h" + +#include namespace td { string HttpUrl::get_url() const { string result; switch (protocol_) { - case Protocol::HTTP: + case Protocol::Http: result += "http://"; break; - case Protocol::HTTPS: + case Protocol::Https: result += "https://"; break; default: @@ -29,39 +32,32 @@ string HttpUrl::get_url() const { result += userinfo_; result += '@'; } - if (is_ipv6) { - result += '['; - } result += host_; - if (is_ipv6) { - result += ']'; - } if (specified_port_ > 0) { result += ':'; result += to_string(specified_port_); } - CHECK(!query_.empty() && query_[0] == '/'); + LOG_CHECK(!query_.empty() && query_[0] == '/') << query_; result += query_; return result; } -Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) { +Result parse_url(Slice url, HttpUrl::Protocol default_protocol) { // url == [https?://][userinfo@]host[:port] - Parser parser(url); - string protocol_str = to_lower(parser.read_till_nofail(':')); + ConstParser parser(url); + string protocol_str = to_lower(parser.read_till_nofail(":/?#@[]")); HttpUrl::Protocol protocol; - if (parser.start_with("://")) { - parser.advance(3); + if (parser.try_skip("://")) { if (protocol_str == "http") { - protocol = HttpUrl::Protocol::HTTP; + protocol = HttpUrl::Protocol::Http; } else if (protocol_str == "https") { - protocol = HttpUrl::Protocol::HTTPS; + protocol = HttpUrl::Protocol::Https; } else { return Status::Error("Unsupported URL protocol"); } } else { - parser = Parser(url); + parser = ConstParser(url); protocol = default_protocol; } Slice userinfo_host_port = parser.read_till_nofail("/?#"); @@ -73,7 +69,16 @@ Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) } Slice userinfo_host; if (colon > userinfo_host_port.begin() && *colon == ':') { - port = to_integer(Slice(colon + 1, userinfo_host_port.end())); + Slice port_slice(colon + 1, userinfo_host_port.end()); + while (port_slice.size() > 1 && port_slice[0] == '0') { + port_slice.remove_prefix(1); + } + auto r_port = to_integer_safe(port_slice); + if (r_port.is_error() || r_port.ok() == 0) { + port = -1; + } else { + port = r_port.ok(); + } userinfo_host = Slice(userinfo_host_port.begin(), colon); } else { userinfo_host = userinfo_host_port; @@ -88,20 +93,26 @@ Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) bool is_ipv6 = false; if (!host.empty() && host[0] == '[' && host.back() == ']') { - host.remove_prefix(1); - host.remove_suffix(1); + IPAddress ip_address; + if (ip_address.init_ipv6_port(host.str(), 1).is_error()) { + return Status::Error("Wrong IPv6 address specified in the URL"); + } + CHECK(ip_address.is_ipv6()); is_ipv6 = true; } if (host.empty()) { return Status::Error("URL host is empty"); } + if (host == ".") { + return Status::Error("Host is invalid"); + } int specified_port = port; if (port == 0) { - if (protocol == HttpUrl::Protocol::HTTP) { + if (protocol == HttpUrl::Protocol::Http) { port = 80; } else { - CHECK(protocol == HttpUrl::Protocol::HTTPS); + CHECK(protocol == HttpUrl::Protocol::Https); port = 443; } } @@ -111,7 +122,7 @@ Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) query.remove_suffix(1); } if (query.empty()) { - query = "/"; + query = Slice("/"); } string query_str; if (query[0] != '/') { @@ -130,6 +141,14 @@ Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) string host_str = to_lower(host); for (size_t i = 0; i < host_str.size(); i++) { char c = host_str[i]; + if (is_ipv6) { + if (i == 0 || i + 1 == host_str.size() || c == ':' || ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || + c == '.') { + continue; + } + return Status::Error("Wrong IPv6 URL host"); + } + if (('a' <= c && c <= 'z') || c == '.' || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '!' || c == '$' || c == ',' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')' || c == ';' || c == '&' || c == '+' || c == '=') { @@ -145,9 +164,11 @@ Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) continue; } } + return Status::Error("Wrong percent-encoded symbol in URL host"); } + // all other symbols aren't allowed - unsigned char uc = static_cast(c); + auto uc = static_cast(c); if (uc >= 128) { // but we allow plain UTF-8 symbols continue; @@ -159,11 +180,66 @@ Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) } StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) { - sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::HTTP ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_) + sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::Http ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_) << tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_); return sb; } +HttpUrlQuery parse_url_query(Slice query) { + if (!query.empty() && query[0] == '/') { + query.remove_prefix(1); + } + + size_t path_size = 0; + while (path_size < query.size() && query[path_size] != '?' && query[path_size] != '#') { + path_size++; + } + + HttpUrlQuery result; + result.path_ = full_split(url_decode(query.substr(0, path_size), false), '/'); + while (!result.path_.empty() && result.path_.back().empty()) { + result.path_.pop_back(); + } + + if (path_size < query.size() && query[path_size] == '?') { + query = query.substr(path_size + 1); + query.truncate(query.find('#')); + + ConstParser parser(query); + while (!parser.data().empty()) { + auto key_value = split(parser.read_till_nofail('&'), '='); + parser.skip_nofail('&'); + auto key = url_decode(key_value.first, true); + if (!key.empty()) { + result.args_.emplace_back(std::move(key), url_decode(key_value.second, true)); + } + } + CHECK(parser.status().is_ok()); + } + + return result; +} + +bool HttpUrlQuery::has_arg(Slice key) const { + auto it = + std::find_if(args_.begin(), args_.end(), [&key](const std::pair &s) { return s.first == key; }); + return it != args_.end(); +} + +Slice HttpUrlQuery::get_arg(Slice key) const { + auto it = + std::find_if(args_.begin(), args_.end(), [&key](const std::pair &s) { return s.first == key; }); + return it == args_.end() ? Slice() : it->second; +} + +string get_url_host(Slice url) { + auto r_http_url = parse_url(url); + if (r_http_url.is_error()) { + return string(); + } + return r_http_url.ok().host_; +} + string get_url_query_file_name(const string &query) { Slice query_slice = query; query_slice.truncate(query.find_first_of("?#")); @@ -175,10 +251,8 @@ string get_url_query_file_name(const string &query) { return query_slice.str(); } -string get_url_file_name(const string &url) { - // TODO remove copy - string url_copy = url; - auto r_http_url = parse_url(url_copy); +string get_url_file_name(Slice url) { + auto r_http_url = parse_url(url); if (r_http_url.is_error()) { LOG(WARNING) << "Receive wrong URL \"" << url << '"'; return string(); diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h index f7d1e4aaba..9b4e92edce 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -11,29 +11,54 @@ #include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include + namespace td { class HttpUrl { public: - enum class Protocol { HTTP, HTTPS } protocol_; + enum class Protocol { Http, Https } protocol_ = Protocol::Http; string userinfo_; string host_; - bool is_ipv6; - int specified_port_; - int port_; + bool is_ipv6_ = false; + int specified_port_ = 0; + int port_ = 0; string query_; string get_url() const; + + HttpUrl(Protocol protocol, string userinfo, string host, bool is_ipv6, int specified_port, int port, string query) + : protocol_(protocol) + , userinfo_(std::move(userinfo)) + , host_(std::move(host)) + , is_ipv6_(is_ipv6) + , specified_port_(specified_port) + , port_(port) + , query_(std::move(query)) { + } }; -// TODO Slice instead of MutableSlice -Result parse_url(MutableSlice url, - HttpUrl::Protocol default_protocol = HttpUrl::Protocol::HTTP) TD_WARN_UNUSED_RESULT; +Result parse_url(Slice url, + HttpUrl::Protocol default_protocol = HttpUrl::Protocol::Http) TD_WARN_UNUSED_RESULT; StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url); +class HttpUrlQuery { + public: + vector path_; + vector> args_; + + bool has_arg(Slice key) const; + + Slice get_arg(Slice key) const; +}; + +HttpUrlQuery parse_url_query(Slice query); + +string get_url_host(Slice url); + string get_url_query_file_name(const string &query); -string get_url_file_name(const string &url); +string get_url_file_name(Slice url); } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp index eb654f43cd..f5823ad806 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -8,10 +8,12 @@ #include "td/utils/misc.h" #include "td/utils/ScopeGuard.h" +#include "td/utils/SliceBuilder.h" #include namespace td { + StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val) { sb << '"'; SCOPE_EXIT { @@ -94,11 +96,11 @@ StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) { break; } if (128 <= ch) { - int a = s[pos]; + uint32 a = ch; CHECK((a & 0x40) != 0); CHECK(pos + 1 < len); - int b = s[++pos]; + uint32 b = static_cast(s[++pos]); CHECK((b & 0xc0) == 0x80); if ((a & 0x20) == 0) { CHECK((a & 0x1e) > 0); @@ -107,7 +109,7 @@ StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) { } CHECK(pos + 1 < len); - int c = s[++pos]; + uint32 c = static_cast(s[++pos]); CHECK((c & 0xc0) == 0x80); if ((a & 0x10) == 0) { CHECK(((a & 0x0f) | (b & 0x20)) > 0); @@ -116,7 +118,7 @@ StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) { } CHECK(pos + 1 < len); - int d = s[++pos]; + uint32 d = static_cast(s[++pos]); CHECK((d & 0xc0) == 0x80); if ((a & 0x08) == 0) { CHECK(((a & 0x07) | (b & 0x30)) > 0); @@ -133,6 +135,7 @@ StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) { } return sb; } + Result json_string_decode(Parser &parser) { if (!parser.try_skip('"')) { return Status::Error("Opening '\"' expected"); @@ -232,7 +235,7 @@ Result json_string_decode(Parser &parser) { } else if (num < 0x800) { *cur_dest++ = static_cast(0xc0 + (num >> 6)); *cur_dest++ = static_cast(0x80 + (num & 63)); - } else if (num < 0xffff) { + } else if (num <= 0xffff) { *cur_dest++ = static_cast(0xe0 + (num >> 12)); *cur_dest++ = static_cast(0x80 + ((num >> 6) & 63)); *cur_dest++ = static_cast(0x80 + (num & 63)); @@ -341,20 +344,20 @@ Result do_json_decode(Parser &parser, int32 max_depth) { parser.skip_whitespaces(); switch (parser.peek_char()) { case 'f': - if (parser.skip_start_with("false")) { + if (parser.try_skip("false")) { return JsonValue::create_boolean(false); } - return Status::Error("Starts with 'f' -- false expected"); + return Status::Error("Token starts with 'f' -- false expected"); case 't': - if (parser.skip_start_with("true")) { + if (parser.try_skip("true")) { return JsonValue::create_boolean(true); } - return Status::Error("Starts with 't' -- true expected"); + return Status::Error("Token starts with 't' -- true expected"); case 'n': - if (parser.skip_start_with("null")) { + if (parser.try_skip("null")) { return JsonValue(); } - return Status::Error("Starts with 'n' -- null expected"); + return Status::Error("Token starts with 'n' -- null expected"); case '"': { TRY_RESULT(slice, json_string_decode(parser)); return JsonValue::create_string(slice); @@ -368,7 +371,7 @@ Result do_json_decode(Parser &parser, int32 max_depth) { } while (true) { if (parser.empty()) { - return Status::Error("Unexpected end"); + return Status::Error("Unexpected string end"); } TRY_RESULT(value, do_json_decode(parser, max_depth - 1)); res.emplace_back(std::move(value)); @@ -381,20 +384,23 @@ Result do_json_decode(Parser &parser, int32 max_depth) { parser.skip_whitespaces(); continue; } - return Status::Error("Unexpected symbol"); + if (parser.empty()) { + return Status::Error("Unexpected string end"); + } + return Status::Error("Unexpected symbol while parsing JSON Array"); } return JsonValue::create_array(std::move(res)); } case '{': { parser.skip('{'); parser.skip_whitespaces(); - std::vector > res; + std::vector> res; if (parser.try_skip('}')) { return JsonValue::make_object(std::move(res)); } while (true) { if (parser.empty()) { - return Status::Error("Unexpected end"); + return Status::Error("Unexpected string end"); } TRY_RESULT(key, json_string_decode(parser)); parser.skip_whitespaces(); @@ -402,7 +408,7 @@ Result do_json_decode(Parser &parser, int32 max_depth) { return Status::Error("':' expected"); } TRY_RESULT(value, do_json_decode(parser, max_depth - 1)); - res.emplace_back(std::move(key), std::move(value)); + res.emplace_back(key, std::move(value)); parser.skip_whitespaces(); if (parser.try_skip('}')) { @@ -412,7 +418,10 @@ Result do_json_decode(Parser &parser, int32 max_depth) { parser.skip_whitespaces(); continue; } - return Status::Error("Unexpected symbol"); + if (parser.empty()) { + return Status::Error("Unexpected string end"); + } + return Status::Error("Unexpected symbol while parsing JSON Object"); } return JsonValue::make_object(std::move(res)); } @@ -434,7 +443,7 @@ Result do_json_decode(Parser &parser, int32 max_depth) { return JsonValue::create_number(num); } case 0: - return Status::Error("Unexpected end"); + return Status::Error("Unexpected string end"); default: { char next = parser.peek_char(); if (0 < next && next < 127) { @@ -455,17 +464,17 @@ Status do_json_skip(Parser &parser, int32 max_depth) { parser.skip_whitespaces(); switch (parser.peek_char()) { case 'f': - if (parser.skip_start_with("false")) { + if (parser.try_skip("false")) { return Status::OK(); } return Status::Error("Starts with 'f' -- false expected"); case 't': - if (parser.skip_start_with("true")) { + if (parser.try_skip("true")) { return Status::OK(); } return Status::Error("Starts with 't' -- true expected"); case 'n': - if (parser.skip_start_with("null")) { + if (parser.try_skip("null")) { return Status::OK(); } return Status::Error("Starts with 'n' -- null expected"); @@ -576,7 +585,7 @@ Slice JsonValue::get_type_name(Type type) { } } -bool has_json_object_field(JsonObject &object, Slice name) { +bool has_json_object_field(const JsonObject &object, Slice name) { for (auto &field_value : object) { if (field_value.first == name) { return true; @@ -585,6 +594,15 @@ bool has_json_object_field(JsonObject &object, Slice name) { return false; } +JsonValue get_json_object_field_force(JsonObject &object, Slice name) { + for (auto &field_value : object) { + if (field_value.first == name) { + return std::move(field_value.second); + } + } + return JsonValue(); +} + Result get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, bool is_optional) { for (auto &field_value : object) { if (field_value.first == name) { @@ -611,11 +629,41 @@ Result get_json_object_bool_field(JsonObject &object, Slice name, bool is_ } Result get_json_object_int_field(JsonObject &object, Slice name, bool is_optional, int32 default_value) { - TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Number, is_optional)); - if (value.type() == JsonValue::Type::Null) { + for (auto &field_value : object) { + if (field_value.first == name) { + if (field_value.second.type() == JsonValue::Type::String) { + return to_integer_safe(field_value.second.get_string()); + } + if (field_value.second.type() == JsonValue::Type::Number) { + return to_integer_safe(field_value.second.get_number()); + } + + return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type Number"); + } + } + if (is_optional) { return default_value; } - return to_integer_safe(value.get_number()); + return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\""); +} + +Result get_json_object_long_field(JsonObject &object, Slice name, bool is_optional, int64 default_value) { + for (auto &field_value : object) { + if (field_value.first == name) { + if (field_value.second.type() == JsonValue::Type::String) { + return to_integer_safe(field_value.second.get_string()); + } + if (field_value.second.type() == JsonValue::Type::Number) { + return to_integer_safe(field_value.second.get_number()); + } + + return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be a Number"); + } + } + if (is_optional) { + return default_value; + } + return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\""); } Result get_json_object_double_field(JsonObject &object, Slice name, bool is_optional, double default_value) { diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h index 735c4b29ec..90035cd6e2 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -15,20 +15,11 @@ #include "td/utils/StringBuilder.h" #include -#include #include #include namespace td { -template -std::tuple ctie(const Args &... args) TD_WARN_UNUSED_RESULT; - -template -std::tuple ctie(const Args &... args) { - return std::tie(args...); -} - class JsonTrue { public: friend StringBuilder &operator<<(StringBuilder &sb, const JsonTrue &val) { @@ -104,7 +95,7 @@ class JsonFloat { class JsonOneChar { public: - explicit JsonOneChar(unsigned int c) : c_(c) { + explicit JsonOneChar(uint32 c) : c_(c) { } friend StringBuilder &operator<<(StringBuilder &sb, const JsonOneChar &val) { @@ -114,12 +105,12 @@ class JsonOneChar { } private: - unsigned int c_; + uint32 c_; }; class JsonChar { public: - explicit JsonChar(unsigned int c) : c_(c) { + explicit JsonChar(uint32 c) : c_(c) { } friend StringBuilder &operator<<(StringBuilder &sb, const JsonChar &val) { auto c = val.c_; @@ -138,7 +129,7 @@ class JsonChar { } private: - unsigned int c_; + uint32 c_; }; class JsonRaw { @@ -181,7 +172,7 @@ class JsonObjectScope; class JsonBuilder { public: - explicit JsonBuilder(StringBuilder &&sb) : sb_(std::move(sb)) { + explicit JsonBuilder(StringBuilder &&sb = {}, int32 offset = -1) : sb_(std::move(sb)), offset_(offset) { } StringBuilder &string_builder() { return sb_; @@ -191,22 +182,48 @@ class JsonBuilder { JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT; JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT; + int32 offset() const { + return offset_; + } + bool is_pretty() const { + return offset_ >= 0; + } + void print_offset() { + if (offset_ >= 0) { + sb_ << '\n'; + for (int x = 0; x < offset_; x++) { + sb_ << " "; + } + } + } + void dec_offset() { + if (offset_ >= 0) { + CHECK(offset_ > 0); + offset_--; + } + } + void inc_offset() { + if (offset_ >= 0) { + offset_++; + } + } + private: StringBuilder sb_; JsonScope *scope_ = nullptr; + int32 offset_; }; class Jsonable {}; class JsonScope { public: - explicit JsonScope(JsonBuilder *jb) : sb_(&jb->sb_), jb_(jb) { - save_scope_ = jb_->scope_; + explicit JsonScope(JsonBuilder *jb) : sb_(&jb->sb_), jb_(jb), save_scope_(jb->scope_) { jb_->scope_ = this; CHECK(is_active()); } JsonScope(const JsonScope &other) = delete; - JsonScope(JsonScope &&other) : sb_(other.sb_), jb_(other.jb_), save_scope_(other.save_scope_) { + JsonScope(JsonScope &&other) noexcept : sb_(other.sb_), jb_(other.jb_), save_scope_(other.save_scope_) { other.jb_ = nullptr; } JsonScope &operator=(const JsonScope &) = delete; @@ -272,9 +289,7 @@ class JsonScope { *sb_ << x; return *this; } - JsonScope &operator<<(bool x) { - return *this << JsonBool(x); - } + JsonScope &operator<<(bool x) = delete; JsonScope &operator<<(int32 x) { return *this << JsonInt(x); } @@ -284,8 +299,6 @@ class JsonScope { JsonScope &operator<<(double x) { return *this << JsonFloat(x); } - template - JsonScope &operator<<(const T *x); // not implemented template JsonScope &operator<<(const char (&x)[N]) { return *this << JsonString(Slice(x)); @@ -293,15 +306,12 @@ class JsonScope { JsonScope &operator<<(const char *x) { return *this << JsonString(Slice(x)); } - JsonScope &operator<<(const string &x) { - return *this << JsonString(Slice(x)); - } JsonScope &operator<<(Slice x) { return *this << JsonString(x); } }; -class JsonValueScope : public JsonScope { +class JsonValueScope final : public JsonScope { public: using JsonScope::JsonScope; template @@ -326,9 +336,10 @@ class JsonValueScope : public JsonScope { bool was_ = false; }; -class JsonArrayScope : public JsonScope { +class JsonArrayScope final : public JsonScope { public: explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) { + jb->inc_offset(); *sb_ << "["; } JsonArrayScope(JsonArrayScope &&other) = default; @@ -338,10 +349,16 @@ class JsonArrayScope : public JsonScope { } } void leave() { + jb_->dec_offset(); + jb_->print_offset(); *sb_ << "]"; } template JsonArrayScope &operator<<(const T &x) { + return (*this)(x); + } + template + JsonArrayScope &operator()(const T &x) { enter_value() << x; return *this; } @@ -352,6 +369,7 @@ class JsonArrayScope : public JsonScope { } else { is_first_ = true; } + jb_->print_offset(); return jb_->enter_value(); } @@ -359,9 +377,10 @@ class JsonArrayScope : public JsonScope { bool is_first_ = false; }; -class JsonObjectScope : public JsonScope { +class JsonObjectScope final : public JsonScope { public: explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) { + jb->inc_offset(); *sb_ << "{"; } JsonObjectScope(JsonObjectScope &&other) = default; @@ -371,23 +390,26 @@ class JsonObjectScope : public JsonScope { } } void leave() { + jb_->dec_offset(); + jb_->print_offset(); *sb_ << "}"; } - template - JsonObjectScope &operator<<(std::tuple key_value) { - return *this << std::pair(std::get<0>(key_value), std::get<1>(key_value)); - } - template - JsonObjectScope &operator<<(std::pair key_value) { + template + JsonObjectScope &operator()(Slice key, T &&value) { CHECK(is_active()); if (is_first_) { *sb_ << ","; } else { is_first_ = true; } - jb_->enter_value() << key_value.first; - *sb_ << ":"; - jb_->enter_value() << key_value.second; + jb_->print_offset(); + jb_->enter_value() << key; + if (jb_->is_pretty()) { + *sb_ << " : "; + } else { + *sb_ << ":"; + } + jb_->enter_value() << value; return *this; } JsonObjectScope &operator<<(const JsonRaw &key_value) { @@ -426,7 +448,7 @@ class JsonValue; using JsonObject = vector>; using JsonArray = vector; -class JsonValue : public Jsonable { +class JsonValue final : private Jsonable { public: enum class Type { Null, Number, Boolean, String, Array, Object }; @@ -437,10 +459,10 @@ class JsonValue : public Jsonable { ~JsonValue() { destroy(); } - JsonValue(JsonValue &&other) : JsonValue() { + JsonValue(JsonValue &&other) noexcept : JsonValue() { init(std::move(other)); } - JsonValue &operator=(JsonValue &&other) { + JsonValue &operator=(JsonValue &&other) noexcept { if (&other == this) { return *this; } @@ -557,7 +579,7 @@ class JsonValue : public Jsonable { case Type::Object: { auto object = scope->enter_object(); for (auto &key_value : get_object()) { - object << ctie(JsonString(key_value.first), key_value.second); + object(key_value.first, key_value.second); } break; } @@ -645,25 +667,25 @@ class JsonValue : public Jsonable { inline StringBuilder &operator<<(StringBuilder &sb, JsonValue::Type type) { switch (type) { - case JsonValue::Type::Object: - return sb << "JsonObject"; - case JsonValue::Type::Boolean: - return sb << "JsonBoolean"; case JsonValue::Type::Null: - return sb << "JsonNull"; + return sb << "Null"; case JsonValue::Type::Number: - return sb << "JsonNumber"; - case JsonValue::Type::Array: - return sb << "JsonArray"; + return sb << "Number"; + case JsonValue::Type::Boolean: + return sb << "Boolean"; case JsonValue::Type::String: - return sb << "JsonString"; + return sb << "String"; + case JsonValue::Type::Array: + return sb << "Array"; + case JsonValue::Type::Object: + return sb << "Object"; default: UNREACHABLE(); return sb; } } -class VirtuallyJsonable : public Jsonable { +class VirtuallyJsonable : private Jsonable { public: virtual void store(JsonValueScope *scope) const = 0; VirtuallyJsonable() = default; @@ -674,11 +696,11 @@ class VirtuallyJsonable : public Jsonable { virtual ~VirtuallyJsonable() = default; }; -class VirtuallyJsonableInt : public VirtuallyJsonable { +class VirtuallyJsonableInt final : public VirtuallyJsonable { public: explicit VirtuallyJsonableInt(int32 value) : value_(value) { } - void store(JsonValueScope *scope) const override { + void store(JsonValueScope *scope) const final { *scope << JsonInt(value_); } @@ -686,11 +708,11 @@ class VirtuallyJsonableInt : public VirtuallyJsonable { int32 value_; }; -class VirtuallyJsonableLong : public VirtuallyJsonable { +class VirtuallyJsonableLong final : public VirtuallyJsonable { public: explicit VirtuallyJsonableLong(int64 value) : value_(value) { } - void store(JsonValueScope *scope) const override { + void store(JsonValueScope *scope) const final { *scope << JsonLong(value_); } @@ -698,11 +720,11 @@ class VirtuallyJsonableLong : public VirtuallyJsonable { int64 value_; }; -class VirtuallyJsonableString : public VirtuallyJsonable { +class VirtuallyJsonableString final : public VirtuallyJsonable { public: explicit VirtuallyJsonableString(Slice value) : value_(value) { } - void store(JsonValueScope *scope) const override { + void store(JsonValueScope *scope) const final { *scope << JsonString(value_); } @@ -716,8 +738,8 @@ Status json_string_skip(Parser &parser) TD_WARN_UNUSED_RESULT; Result do_json_decode(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT; Status do_json_skip(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT; -inline Result json_decode(MutableSlice from) { - Parser parser(from); +inline Result json_decode(MutableSlice json) { + Parser parser(json); const int32 DEFAULT_MAX_DEPTH = 100; auto result = do_json_decode(parser, DEFAULT_MAX_DEPTH); if (result.is_ok()) { @@ -730,17 +752,92 @@ inline Result json_decode(MutableSlice from) { } template -StrT json_encode(const ValT &val) { - auto buf_len = 1 << 19; +StrT json_encode(const ValT &val, bool pretty = false) { + auto buf_len = 1 << 18; auto buf = StackAllocator::alloc(buf_len); - JsonBuilder jb(StringBuilder(buf.as_slice())); + JsonBuilder jb(StringBuilder(buf.as_slice(), true), pretty ? 0 : -1); jb.enter_value() << val; - LOG_IF(ERROR, jb.string_builder().is_error()) << "Json buffer overflow"; + if (pretty) { + jb.string_builder() << "\n"; + } + LOG_IF(ERROR, jb.string_builder().is_error()) << "JSON buffer overflow"; auto slice = jb.string_builder().as_cslice(); return StrT(slice.begin(), slice.size()); } -bool has_json_object_field(JsonObject &object, Slice name); +template +class ToJsonImpl final : private Jsonable { + public: + explicit ToJsonImpl(const T &value) : value_(value) { + } + void store(JsonValueScope *scope) const { + to_json(*scope, value_); + } + + private: + const T &value_; +}; + +template +auto ToJson(const T &value) { + return ToJsonImpl(value); +} + +template +void to_json(JsonValueScope &jv, const T &value) { + jv << value; +} + +template +class JsonObjectImpl : private Jsonable { + public: + explicit JsonObjectImpl(F &&f) : f_(std::forward(f)) { + } + void store(JsonValueScope *scope) const { + auto object = scope->enter_object(); + f_(object); + } + + private: + F f_; +}; + +template +auto json_object(F &&f) { + return JsonObjectImpl(std::forward(f)); +} + +template +class JsonArrayImpl : private Jsonable { + public: + explicit JsonArrayImpl(F &&f) : f_(std::forward(f)) { + } + void store(JsonValueScope *scope) const { + auto array = scope->enter_array(); + f_(array); + } + + private: + F f_; +}; + +template +auto json_array(F &&f) { + return JsonArrayImpl(std::forward(f)); +} + +template +auto json_array(const A &a, F &&f) { + return json_array([&a, &f](auto &arr) { + for (auto &x : a) { + arr(f(x)); + } + }); +} + +bool has_json_object_field(const JsonObject &object, Slice name); + +JsonValue get_json_object_field_force(JsonObject &object, Slice name) TD_WARN_UNUSED_RESULT; Result get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, bool is_optional = true) TD_WARN_UNUSED_RESULT; @@ -751,6 +848,9 @@ Result get_json_object_bool_field(JsonObject &object, Slice name, bool is_ Result get_json_object_int_field(JsonObject &object, Slice name, bool is_optional = true, int32 default_value = 0) TD_WARN_UNUSED_RESULT; +Result get_json_object_long_field(JsonObject &object, Slice name, bool is_optional = true, + int64 default_value = 0) TD_WARN_UNUSED_RESULT; + Result get_json_object_double_field(JsonObject &object, Slice name, bool is_optional = true, double default_value = 0.0) TD_WARN_UNUSED_RESULT; diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h index 1606c44d2b..4f9bb9877f 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h @@ -1,12 +1,12 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #pragma once -#include "td/utils/logging.h" +#include "td/utils/common.h" namespace td { @@ -24,23 +24,23 @@ struct ListNode { ListNode(const ListNode &) = delete; ListNode &operator=(const ListNode &) = delete; - ListNode(ListNode &&other) { + ListNode(ListNode &&other) noexcept { if (other.empty()) { clear(); } else { - ListNode *head = other.prev; - other.remove(); - head->put(this); + init_from(std::move(other)); } } - ListNode &operator=(ListNode &&other) { + ListNode &operator=(ListNode &&other) noexcept { + if (this == &other) { + return *this; + } + this->remove(); if (!other.empty()) { - ListNode *head = other.prev; - other.remove(); - head->put(this); + init_from(std::move(other)); } return *this; @@ -58,11 +58,12 @@ struct ListNode { } void put(ListNode *other) { - other->connect(next); - this->connect(other); + DCHECK(other->empty()); + put_unsafe(other); } void put_back(ListNode *other) { + DCHECK(other->empty()); prev->connect(other); other->connect(this); } @@ -82,11 +83,47 @@ struct ListNode { return next == this; } - private: + ListNode *begin() { + return next; + } + ListNode *end() { + return this; + } + const ListNode *begin() const { + return next; + } + const ListNode *end() const { + return this; + } + ListNode *get_next() { + return next; + } + ListNode *get_prev() { + return prev; + } + const ListNode *get_next() const { + return next; + } + const ListNode *get_prev() const { + return prev; + } + + protected: void clear() { next = this; prev = this; } + + void init_from(ListNode &&other) { + ListNode *head = other.prev; + other.remove(); + head->put_unsafe(this); + } + + void put_unsafe(ListNode *other) { + other->connect(next); + this->connect(other); + } }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MapNode.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MapNode.h new file mode 100644 index 0000000000..cad2ae9b36 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MapNode.h @@ -0,0 +1,166 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/HashTableUtils.h" + +#include +#include +#include + +namespace td { + +template +struct MapNode { + using first_type = KeyT; + using second_type = ValueT; + using public_key_type = KeyT; + using public_type = MapNode; + + KeyT first{}; + union { + ValueT second; + }; + + const KeyT &key() const { + return first; + } + + MapNode &get_public() { + return *this; + } + + const MapNode &get_public() const { + return *this; + } + + MapNode() { + } + MapNode(KeyT key, ValueT value) : first(std::move(key)) { + new (&second) ValueT(std::move(value)); + DCHECK(!empty()); + } + MapNode(const MapNode &other) = delete; + MapNode &operator=(const MapNode &other) = delete; + MapNode(MapNode &&other) noexcept { + *this = std::move(other); + } + void operator=(MapNode &&other) noexcept { + DCHECK(empty()); + DCHECK(!other.empty()); + first = std::move(other.first); + other.first = KeyT(); + new (&second) ValueT(std::move(other.second)); + other.second.~ValueT(); + } + ~MapNode() { + if (!empty()) { + second.~ValueT(); + } + } + + void copy_from(const MapNode &other) { + DCHECK(empty()); + DCHECK(!other.empty()); + first = other.first; + new (&second) ValueT(other.second); + } + + bool empty() const { + return is_hash_table_key_empty(first); + } + + void clear() { + DCHECK(!empty()); + first = KeyT(); + second.~ValueT(); + DCHECK(empty()); + } + + template + void emplace(KeyT key, ArgsT &&...args) { + DCHECK(empty()); + first = std::move(key); + new (&second) ValueT(std::forward(args)...); + DCHECK(!empty()); + } +}; + +template +struct MapNode 28 * sizeof(void *))>> { + struct Impl { + using first_type = KeyT; + using second_type = ValueT; + + KeyT first{}; + union { + ValueT second; + }; + + template + Impl(InputKeyT &&key, ArgsT &&...args) : first(std::forward(key)) { + new (&second) ValueT(std::forward(args)...); + DCHECK(!is_hash_table_key_empty(first)); + } + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + void operator=(Impl &&other) = delete; + ~Impl() { + second.~ValueT(); + } + }; + + using first_type = KeyT; + using second_type = ValueT; + using public_key_type = KeyT; + using public_type = Impl; + + unique_ptr impl_; + + const KeyT &key() const { + DCHECK(!empty()); + return impl_->first; + } + + Impl &get_public() { + return *impl_; + } + + const Impl &get_public() const { + return *impl_; + } + + MapNode() { + } + MapNode(KeyT key, ValueT value) : impl_(td::make_unique(std::move(key), std::move(value))) { + } + + void copy_from(const MapNode &other) { + DCHECK(empty()); + DCHECK(!other.empty()); + impl_ = td::make_unique(other.impl_->first, other.impl_->second); + } + + bool empty() const { + return impl_ == nullptr; + } + + void clear() { + DCHECK(!empty()); + impl_ = nullptr; + } + + template + void emplace(KeyT key, ArgsT &&...args) { + DCHECK(empty()); + impl_ = td::make_unique(std::move(key), std::forward(args)...); + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h index aa125df2f7..04b03b60ee 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -17,15 +17,27 @@ namespace td { template -class MemoryLog : public LogInterface { +class MemoryLog final : public LogInterface { static constexpr size_t MAX_OUTPUT_SIZE = buffer_size / 16 < (8 << 10) ? buffer_size / 16 : (8 << 10); + static_assert((buffer_size & (buffer_size - 1)) == 0, "Buffer size must be power of 2"); + static_assert(buffer_size >= (8 << 10), "Too small buffer size"); + public: MemoryLog() { std::memset(buffer_, ' ', sizeof(buffer_)); } - void append(CSlice new_slice, int log_level) override { + Slice get_buffer() const { + return Slice(buffer_, sizeof(buffer_)); + } + + size_t get_pos() const { + return pos_ & (buffer_size - 1); + } + + private: + void do_append(int log_level, CSlice new_slice) final { Slice slice = new_slice; slice.truncate(MAX_OUTPUT_SIZE); while (!slice.empty() && slice.back() == '\n') { @@ -34,50 +46,34 @@ class MemoryLog : public LogInterface { size_t slice_size = slice.size(); CHECK(slice_size * 3 < buffer_size); size_t pad_size = ((slice_size + 15) & ~15) - slice_size; - constexpr size_t magic_size = 16; - uint32 total_size = static_cast(slice_size + pad_size + magic_size); - uint32 real_pos = pos_.fetch_add(total_size, std::memory_order_relaxed); + constexpr size_t MAGIC_SIZE = 16; + auto total_size = static_cast(slice_size + pad_size + MAGIC_SIZE); + auto real_pos = pos_.fetch_add(total_size, std::memory_order_relaxed); CHECK((total_size & 15) == 0); uint32 start_pos = real_pos & (buffer_size - 1); uint32 end_pos = start_pos + total_size; if (likely(end_pos <= buffer_size)) { - std::memcpy(&buffer_[start_pos + magic_size], slice.data(), slice_size); - std::memcpy(&buffer_[start_pos + magic_size + slice_size], " ", pad_size); + std::memcpy(&buffer_[start_pos + MAGIC_SIZE], slice.data(), slice_size); + std::memcpy(&buffer_[start_pos + MAGIC_SIZE + slice_size], " ", pad_size); } else { - size_t first = buffer_size - start_pos - magic_size; + size_t first = buffer_size - start_pos - MAGIC_SIZE; size_t second = slice_size - first; - std::memcpy(&buffer_[start_pos + magic_size], slice.data(), first); + std::memcpy(&buffer_[start_pos + MAGIC_SIZE], slice.data(), first); std::memcpy(&buffer_[0], slice.data() + first, second); - std::memcpy(&buffer_[second], " ", pad_size); + std::memcpy(&buffer_[second], " ", pad_size); } CHECK((start_pos & 15) == 0); - CHECK(start_pos <= buffer_size - magic_size); + CHECK(start_pos <= buffer_size - MAGIC_SIZE); buffer_[start_pos] = '\n'; - size_t printed = std::snprintf(&buffer_[start_pos + 1], magic_size - 1, "LOG:%08x: ", real_pos); - CHECK(printed == magic_size - 2); - buffer_[start_pos + magic_size - 1] = ' '; - - if (log_level == VERBOSITY_NAME(FATAL)) { - process_fatal_error(new_slice); - } + size_t printed = std::snprintf(&buffer_[start_pos + 1], MAGIC_SIZE - 1, "LOG:%08x: ", real_pos); + CHECK(printed == MAGIC_SIZE - 2); + buffer_[start_pos + MAGIC_SIZE - 1] = ' '; } - void rotate() override { - } - - Slice get_buffer() const { - return Slice(buffer_, sizeof(buffer_)); - } - - size_t get_pos() const { - return pos_ & (buffer_size - 1); - } - - private: char buffer_[buffer_size]; - std::atomic pos_; + std::atomic pos_{0}; }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp index 75c4fe34b5..a7dde2405f 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h index 11210ceb30..ccaf029a52 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h index 939bf51f28..7a8ef459e0 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -14,16 +14,18 @@ class MovableValue { MovableValue() = default; MovableValue(T val) : val_(val) { } - MovableValue(MovableValue &&other) : val_(other.val_) { + MovableValue(MovableValue &&other) noexcept : val_(other.val_) { other.clear(); } - MovableValue &operator=(MovableValue &&other) { - val_ = other.val_; - other.clear(); + MovableValue &operator=(MovableValue &&other) noexcept { + if (this != &other) { + val_ = other.val_; + other.clear(); + } return *this; } - MovableValue(const MovableValue &) = delete; - MovableValue &operator=(const MovableValue &) = delete; + MovableValue(const MovableValue &) = default; + MovableValue &operator=(const MovableValue &) = default; ~MovableValue() = default; void clear() { diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.cpp new file mode 100644 index 0000000000..1fcbc04341 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.cpp @@ -0,0 +1,15 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/MpmcQueue.h" + +namespace td { +namespace detail { + +MpmcStat stat_; + +} // namespace detail +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h index ae65554b72..f1f12a37f4 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -14,7 +14,7 @@ #include "td/utils/format.h" #include "td/utils/HazardPointers.h" #include "td/utils/logging.h" -#include "td/utils/port/thread.h" +#include "td/utils/port/sleep.h" #include "td/utils/ScopeGuard.h" #include @@ -23,6 +23,7 @@ namespace td { namespace detail { + struct MpmcStat { void alloc_ok(size_t thread_id) { s(thread_id).alloc_ok_cnt++; @@ -64,8 +65,10 @@ struct MpmcStat { return arr[thread_id]; } }; + +extern MpmcStat stat_; + } // namespace detail -//detail::MpmcStat stat_; template class OneValue { @@ -95,31 +98,35 @@ class OneValue { private: enum Type : int { Empty = 0, Taken, Value }; std::atomic state_{Empty}; - T value_; + T value_{}; }; template class OneValue { public: bool set_value(T *value) { - T *was = nullptr; + T *was = Empty(); return state_.compare_exchange_strong(was, value, std::memory_order_acq_rel); } bool get_value(T *&value) { value = state_.exchange(Taken(), std::memory_order_acq_rel); - return value != nullptr; + return value != Empty(); } void reset() { - state_ = nullptr; + state_ = Empty(); } OneValue() { } private: - std::atomic state_{nullptr}; - T *Taken() { - static T xxx; - return &xxx; + std::atomic state_{Empty()}; + static T *Empty() { + static int64 xxx; + return reinterpret_cast(&xxx); + } + static T *Taken() { + static int64 xxx; + return reinterpret_cast(&xxx); } }; @@ -149,6 +156,11 @@ class MpmcQueueBlock { //returns Ok, Empty or Closed PopStatus try_pop(T &value) { while (true) { + // this check slows 1:1 case but prevents writer starvation in 1:N case + if (write_pos_.load(std::memory_order_relaxed) <= read_pos_.load(std::memory_order_relaxed) && + read_pos_.load(std::memory_order_relaxed) < nodes_.size()) { + return PopStatus::Empty; + } auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed); if (read_pos >= nodes_.size()) { return PopStatus::Closed; @@ -199,7 +211,7 @@ class MpmcQueueOld { return "Mpmc queue (fetch and add array queue)"; } MpmcQueueOld(size_t block_size, size_t threads_n) : block_size_{block_size}, hazard_pointers_{threads_n} { - auto node = std::make_unique(block_size_); + auto node = make_unique(block_size_); write_pos_ = node.get(); read_pos_ = node.get(); node.release(); @@ -217,7 +229,7 @@ class MpmcQueueOld { delete to_delete; } //stat_.dump(); - //stat_ = MpmcStat(); + //stat_ = detail::MpmcStat(); } size_t hazard_pointers_to_delele_size_unsafe() const { @@ -231,7 +243,7 @@ class MpmcQueueOld { using PopStatus = typename MpmcQueueBlock::PopStatus; void push(T value, size_t thread_id) { - auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0); + typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0); while (true) { auto node = hazard_ptr_holder.protect(write_pos_); auto status = node->block.push(value); @@ -263,7 +275,7 @@ class MpmcQueueOld { } bool try_pop(T &value, size_t thread_id) { - auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0); + typename decltype(hazard_pointers_)::Holder hazard_ptr_holder(hazard_pointers_, thread_id, 0); while (true) { auto node = hazard_ptr_holder.protect(read_pos_); auto status = node->block.try_pop(value); @@ -293,7 +305,7 @@ class MpmcQueueOld { if (try_pop(value, thread_id)) { return value; } - td::this_thread::yield(); + usleep_for(1); } } @@ -306,9 +318,9 @@ class MpmcQueueOld { MpmcQueueBlock block; //Got pad in MpmcQueueBlock }; - std::atomic write_pos_; + std::atomic write_pos_{nullptr}; char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; - std::atomic read_pos_; + std::atomic read_pos_{nullptr}; char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; size_t block_size_; HazardPointers hazard_pointers_; @@ -324,7 +336,7 @@ class MpmcQueue { return "NEW Mpmc queue (fetch and add array queue)"; } MpmcQueue(size_t block_size, size_t threads_n) : hazard_pointers_{threads_n} { - auto node = std::make_unique(); + auto node = make_unique(); write_pos_ = node.get(); read_pos_ = node.get(); node.release(); @@ -417,7 +429,7 @@ class MpmcQueue { if (try_pop(value, thread_id)) { return value; } - td::this_thread::yield(); + usleep_for(1); } } @@ -438,9 +450,9 @@ class MpmcQueue { char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; //Got pad in MpmcQueueBlock }; - std::atomic write_pos_; + std::atomic write_pos_{nullptr}; char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; - std::atomic read_pos_; + std::atomic read_pos_{nullptr}; char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; HazardPointers hazard_pointers_; //Got pad in HazardPointers diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h index 0f48620e63..7ece498ff6 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,61 +7,81 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/port/thread.h" +#include "td/utils/logging.h" +#include "td/utils/port/sleep.h" +#include #include #include #include namespace td { -class MpmcWaiter { +class MpmcEagerWaiter { public: - int wait(int yields, uint32 worker_id) { - if (yields < RoundsTillSleepy) { - td::this_thread::yield(); - return yields + 1; - } else if (yields == RoundsTillSleepy) { + struct Slot { + private: + friend class MpmcEagerWaiter; + int yields; + uint32 worker_id; + }; + static void init_slot(Slot &slot, uint32 worker_id) { + slot.yields = 0; + slot.worker_id = worker_id; + } + + void wait(Slot &slot) { + if (slot.yields < RoundsTillSleepy) { + yield(); + slot.yields++; + } else if (slot.yields == RoundsTillSleepy) { auto state = state_.load(std::memory_order_relaxed); if (!State::has_worker(state)) { - auto new_state = State::with_worker(state, worker_id); - if (state_.compare_exchange_strong(state, new_state)) { - td::this_thread::yield(); - return yields + 1; + auto new_state = State::with_worker(state, slot.worker_id); + if (state_.compare_exchange_strong(state, new_state, std::memory_order_acq_rel)) { + yield(); + slot.yields++; + return; } if (state == State::awake()) { - return 0; + slot.yields = 0; + return; } } - td::this_thread::yield(); - return 0; - } else if (yields < RoundsTillAsleep) { + yield(); + slot.yields = 0; + } else if (slot.yields < RoundsTillAsleep) { auto state = state_.load(std::memory_order_acquire); - if (State::still_sleepy(state, worker_id)) { - td::this_thread::yield(); - return yields + 1; + if (State::still_sleepy(state, slot.worker_id)) { + yield(); + slot.yields++; + return; } - return 0; + slot.yields = 0; } else { auto state = state_.load(std::memory_order_acquire); - if (State::still_sleepy(state, worker_id)) { + if (State::still_sleepy(state, slot.worker_id)) { std::unique_lock lock(mutex_); if (state_.compare_exchange_strong(state, State::asleep(), std::memory_order_acq_rel)) { condition_variable_.wait(lock); } } - return 0; + slot.yields = 0; } } - int stop_wait(int yields, uint32 worker_id) { - if (yields > RoundsTillSleepy) { + void stop_wait(Slot &slot) { + if (slot.yields > RoundsTillSleepy) { notify_cold(); } - return 0; + slot.yields = 0; + } + + void close() { } void notify() { + std::atomic_thread_fence(std::memory_order_seq_cst); if (state_.load(std::memory_order_acquire) == State::awake()) { return; } @@ -90,6 +110,7 @@ class MpmcWaiter { } }; enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 }; + // enum { RoundsTillSleepy = 1, RoundsTillAsleep = 2 }; std::atomic state_{State::awake()}; std::mutex mutex_; std::condition_variable condition_variable_; @@ -101,6 +122,216 @@ class MpmcWaiter { condition_variable_.notify_all(); } } + static void yield() { + // whatever, this is better than sched_yield + usleep_for(1); + } }; +class MpmcSleepyWaiter { + public: + struct Slot { + private: + friend class MpmcSleepyWaiter; + + enum State { Search, Work, Sleep } state_{Work}; + + void park() { + std::unique_lock guard(mutex_); + condition_variable_.wait(guard, [&] { return unpark_flag_; }); + unpark_flag_ = false; + } + + bool cancel_park() { + auto res = unpark_flag_; + unpark_flag_ = false; + return res; + } + + void unpark() { + //TODO: try to unlock guard before notify_all + std::unique_lock guard(mutex_); + unpark_flag_ = true; + condition_variable_.notify_all(); + } + + std::mutex mutex_; + std::condition_variable condition_variable_; + bool unpark_flag_{false}; // TODO: move out of lock + int yield_cnt{0}; + int32 worker_id{0}; + + public: + char padding[TD_CONCURRENCY_PAD]; + }; + + // There are a lot of workers + // Each has a slot + // + // States of a worker: + // - searching for work | Search + // - processing work | Work + // - sleeping | Sleep + // + // When somebody adds a work it calls notify + // + // notify + // if there are workers in search phase do nothing. + // if all workers are awake do nothing + // otherwise wake some random worker + // + // Initially all workers are in Search mode. + // + // When worker found nothing it may try to call wait. + // This may put it in a Sleep for some time. + // After wait returns worker will be in Search state again. + // + // Suppose worker found a work and ready to process it. + // Then it may call stop_wait. This will cause transition from + // Search to Work state. + // + // Main invariant: + // After notify is called there should be at least on worker in Search or Work state. + // If possible - in Search state + // + + static void init_slot(Slot &slot, int32 worker_id) { + slot.state_ = Slot::State::Work; + slot.unpark_flag_ = false; + slot.worker_id = worker_id; + VLOG(waiter) << "Init slot " << worker_id; + } + + static constexpr int VERBOSITY_NAME(waiter) = VERBOSITY_NAME(DEBUG) + 10; + void wait(Slot &slot) { + if (slot.state_ == Slot::State::Work) { + VLOG(waiter) << "Work -> Search"; + state_++; + slot.state_ = Slot::State::Search; + slot.yield_cnt = 0; + return; + } + if (slot.state_ == Slot::Search) { + if (slot.yield_cnt++ < 10 && false) { + // TODO some sleep backoff is possible + return; + } + + slot.state_ = Slot::State::Sleep; + std::unique_lock guard(sleepers_mutex_); + auto state_view = StateView(state_.fetch_add((1 << PARKING_SHIFT) - 1)); + CHECK(state_view.searching_count != 0); + bool should_search = state_view.searching_count == 1; + if (closed_) { + return; + } + sleepers_.push_back(&slot); + LOG_CHECK(slot.unpark_flag_ == false) << slot.worker_id; + VLOG(waiter) << "Add to sleepers " << slot.worker_id; + //guard.unlock(); + if (should_search) { + VLOG(waiter) << "Search -> Search once, then Sleep "; + return; + } + VLOG(waiter) << "Search -> Sleep " << state_view.searching_count << " " << state_view.parked_count; + } + + CHECK(slot.state_ == Slot::State::Sleep); + VLOG(waiter) << "Park " << slot.worker_id; + slot.park(); + VLOG(waiter) << "Resume " << slot.worker_id; + slot.state_ = Slot::State::Search; + slot.yield_cnt = 0; + } + + void stop_wait(Slot &slot) { + if (slot.state_ == Slot::State::Work) { + return; + } + if (slot.state_ == Slot::State::Sleep) { + VLOG(waiter) << "Search once, then Sleep -> Work/Search " << slot.worker_id; + slot.state_ = Slot::State::Work; + std::unique_lock guard(sleepers_mutex_); + auto it = std::find(sleepers_.begin(), sleepers_.end(), &slot); + if (it != sleepers_.end()) { + sleepers_.erase(it); + VLOG(waiter) << "Remove from sleepers " << slot.worker_id; + state_.fetch_sub((1 << PARKING_SHIFT) - 1); + guard.unlock(); + } else { + guard.unlock(); + VLOG(waiter) << "Not in sleepers" << slot.worker_id; + CHECK(slot.cancel_park()); + } + } + VLOG(waiter) << "Search once, then Sleep -> Work " << slot.worker_id; + slot.state_ = Slot::State::Search; + auto state_view = StateView(state_.fetch_sub(1)); + CHECK(state_view.searching_count != 0); + CHECK(state_view.searching_count < 1000); + bool should_notify = state_view.searching_count == 1; + if (should_notify) { + VLOG(waiter) << "Notify others"; + notify(); + } + VLOG(waiter) << "Search -> Work "; + slot.state_ = Slot::State::Work; + } + + void notify() { + auto view = StateView(state_.load()); + //LOG(ERROR) << view.parked_count; + if (view.searching_count > 0 || view.parked_count == 0) { + VLOG(waiter) << "Ingore notify: " << view.searching_count << " " << view.parked_count; + return; + } + + VLOG(waiter) << "Notify: " << view.searching_count << " " << view.parked_count; + std::unique_lock guard(sleepers_mutex_); + + view = StateView(state_.load()); + if (view.searching_count > 0) { + VLOG(waiter) << "Skip notify: got searching"; + return; + } + + CHECK(view.parked_count == static_cast(sleepers_.size())); + if (sleepers_.empty()) { + VLOG(waiter) << "Skip notify: no sleepers"; + return; + } + + auto sleeper = sleepers_.back(); + sleepers_.pop_back(); + state_.fetch_sub((1 << PARKING_SHIFT) - 1); + VLOG(waiter) << "Unpark " << sleeper->worker_id; + sleeper->unpark(); + } + + void close() { + StateView state(state_.load()); + LOG_CHECK(state.parked_count == 0) << state.parked_count; + LOG_CHECK(state.searching_count == 0) << state.searching_count; + } + + private: + static constexpr int32 PARKING_SHIFT = 16; + struct StateView { + int32 parked_count; + int32 searching_count; + explicit StateView(int32 x) { + parked_count = x >> PARKING_SHIFT; + searching_count = x & ((1 << PARKING_SHIFT) - 1); + } + }; + std::atomic state_{0}; + + std::mutex sleepers_mutex_; + vector sleepers_; + + bool closed_ = false; +}; + +using MpmcWaiter = MpmcSleepyWaiter; + } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h index 4398c7503d..a1b05c2791 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -7,7 +7,6 @@ #pragma once #include "td/utils/common.h" -#include "td/utils/logging.h" #include @@ -149,17 +148,17 @@ template class MpscLinkQueueUniquePtrNode { public: MpscLinkQueueUniquePtrNode() = default; - explicit MpscLinkQueueUniquePtrNode(std::unique_ptr ptr) : ptr_(std::move(ptr)) { + explicit MpscLinkQueueUniquePtrNode(unique_ptr ptr) : ptr_(std::move(ptr)) { } MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() { return ptr_.release()->to_mpsc_link_queue_node(); } - static MpscLinkQueueUniquePtrNode from_mpsc_link_queue_node(td::MpscLinkQueueImpl::Node *node) { - return MpscLinkQueueUniquePtrNode(std::unique_ptr(Value::from_mpsc_link_queue_node(node))); + static MpscLinkQueueUniquePtrNode from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) { + return MpscLinkQueueUniquePtrNode(unique_ptr(Value::from_mpsc_link_queue_node(node))); } - explicit operator bool() { + explicit operator bool() const noexcept { return ptr_ != nullptr; } @@ -168,7 +167,7 @@ class MpscLinkQueueUniquePtrNode { } private: - std::unique_ptr ptr_; + unique_ptr ptr_; }; } // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h index 89d2df8693..f6de4bf280 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h @@ -1,57 +1,63 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #pragma once +#include "td/utils/common.h" #include "td/utils/misc.h" #include "td/utils/port/EventFd.h" -#include "td/utils/SpinLock.h" #if !TD_EVENTFD_UNSUPPORTED -#if !TD_WINDOWS -#include -#include -#endif + +#include "td/utils/port/Mutex.h" #include namespace td { // interface like in PollableQueue -template +template class MpscPollableQueue { public: + using ValueType = T; + int reader_wait_nonblock() { auto ready = reader_vector_.size() - reader_pos_; if (ready != 0) { return narrow_cast(ready); } - auto guard = lock_.lock(); - if (writer_vector_.empty()) { + for (int i = 0; i < 2; i++) { + auto guard = lock_.lock(); + if (writer_vector_.empty()) { + if (i == 1) { + wait_event_fd_ = true; + return 0; + } + } else { + reader_vector_.clear(); + reader_pos_ = 0; + std::swap(writer_vector_, reader_vector_); + return narrow_cast(reader_vector_.size()); + } event_fd_.acquire(); - wait_event_fd_ = true; - return 0; - } else { - reader_vector_.clear(); - reader_pos_ = 0; - std::swap(writer_vector_, reader_vector_); - return narrow_cast(reader_vector_.size()); } + UNREACHABLE(); } - ValueT reader_get_unsafe() { + ValueType reader_get_unsafe() { return std::move(reader_vector_[reader_pos_++]); } void reader_flush() { //nop } - void writer_put(ValueT value) { + void writer_put(ValueType value) { auto guard = lock_.lock(); writer_vector_.push_back(std::move(value)); if (wait_event_fd_) { wait_event_fd_ = false; + guard.reset(); event_fd_.release(); } } @@ -62,6 +68,11 @@ class MpscPollableQueue { //nop } + bool is_empty() { + auto guard = lock_.lock(); + return writer_vector_.empty() && reader_vector_.empty(); + } + void init() { event_fd_.init(); } @@ -75,35 +86,27 @@ class MpscPollableQueue { } } -// Just example of usage -#if !TD_WINDOWS + // Just an example of usage int reader_wait() { int res; - while ((res = reader_wait_nonblock()) == 0) { - // TODO: reader_flush? - pollfd fd; - fd.fd = reader_get_event_fd().get_fd().get_native_fd(); - fd.events = POLLIN; - poll(&fd, 1, -1); + reader_get_event_fd().wait(1000); } return res; } -#endif private: - SpinLock lock_; + Mutex lock_; bool wait_event_fd_{false}; EventFd event_fd_; - std::vector writer_vector_; - std::vector reader_vector_; + std::vector writer_vector_; + std::vector reader_vector_; size_t reader_pos_{0}; }; } // namespace td #else -#include "td/utils/logging.h" namespace td { diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h index 202de5f7d4..251f14b86d 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/NullLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/NullLog.h new file mode 100644 index 0000000000..71b9ad7d7c --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/NullLog.h @@ -0,0 +1,19 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +namespace td { + +class NullLog final : public LogInterface { + void do_append(int /*log_level*/, CSlice /*slice*/) final { + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h index e6e4549dbb..8ee2a58566 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -45,7 +45,7 @@ class ObjectPool { // It is not very usual case of acquire/release use. // Instead of publishing an object via some flag we do the opposite. // We publish new generation via destruction of the data. - // In usual case if we see a flag then we are able to use an object. + // In usual case if we see a flag, then we are able to use an object. // In our case if we have used an object and it is already invalid, then generation will mismatch bool is_alive() const { if (!storage_) { @@ -84,11 +84,11 @@ class ObjectPool { OwnerPtr() = default; OwnerPtr(const OwnerPtr &) = delete; OwnerPtr &operator=(const OwnerPtr &) = delete; - OwnerPtr(OwnerPtr &&other) : storage_(other.storage_), parent_(other.parent_) { + OwnerPtr(OwnerPtr &&other) noexcept : storage_(other.storage_), parent_(other.parent_) { other.storage_ = nullptr; other.parent_ = nullptr; } - OwnerPtr &operator=(OwnerPtr &&other) { + OwnerPtr &operator=(OwnerPtr &&other) noexcept { if (this != &other) { storage_ = other.storage_; parent_ = other.parent_; @@ -156,7 +156,7 @@ class ObjectPool { }; template - OwnerPtr create(ArgsT &&... args) { + OwnerPtr create(ArgsT &&...args) { Storage *storage = get_storage(); storage->init_data(std::forward(args)...); return OwnerPtr(storage, this); @@ -189,7 +189,7 @@ class ObjectPool { delete to_delete; storage_count_--; } - CHECK(storage_count_.load() == 0) << storage_count_.load(); + LOG_CHECK(storage_count_.load() == 0) << storage_count_.load(); } private: @@ -201,7 +201,7 @@ class ObjectPool { std::atomic generation{1}; template - void init_data(ArgsT &&... args) { + void init_data(ArgsT &&...args) { // new (&data) DataT(std::forward(args)...); data = DataT(std::forward(args)...); } diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h index 8511e0ce8b..33e8bc1a4c 100644 --- a/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h @@ -1,5 +1,5 @@ // -// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) @@ -22,13 +22,13 @@ class ObserverBase { virtual void notify() = 0; }; -class Observer : ObserverBase { +class Observer final : private ObserverBase { public: Observer() = default; explicit Observer(unique_ptr &&ptr) : observer_ptr_(std::move(ptr)) { } - void notify() override { + void notify() final { if (observer_ptr_) { observer_ptr_->notify(); } diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.cpp new file mode 100644 index 0000000000..76571d4954 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.cpp @@ -0,0 +1,263 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/OptionParser.h" + +#include "td/utils/FlatHashMap.h" +#include "td/utils/logging.h" +#include "td/utils/PathView.h" +#include "td/utils/SliceBuilder.h" + +#if TD_PORT_WINDOWS +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#include "td/utils/port/wstring_convert.h" +#endif +#endif + +#if TD_PORT_WINDOWS +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#include +#endif +#endif + +namespace td { + +void OptionParser::set_usage(Slice executable_name, Slice usage) { + PathView path_view(executable_name); + usage_ = PSTRING() << path_view.file_name() << " " << usage; +} + +void OptionParser::set_description(string description) { + description_ = std::move(description); +} + +void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, + std::function callback) { + for (auto &option : options_) { + if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) { + LOG(ERROR) << "Ignore duplicated option '" << (short_key == '\0' ? '-' : short_key) << "' '" << long_key << "'"; + } + } + options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)}); +} + +void OptionParser::add_checked_option(char short_key, Slice long_key, Slice description, + std::function callback) { + add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback)); +} + +void OptionParser::add_checked_option(char short_key, Slice long_key, Slice description, + std::function callback) { + add_option(Option::Type::NoArg, short_key, long_key, description, + [callback = std::move(callback)](Slice) { return callback(); }); +} + +void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function callback) { + add_option(Option::Type::Arg, short_key, long_key, description, [callback = std::move(callback)](Slice parameter) { + callback(parameter); + return Status::OK(); + }); +} + +void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function callback) { + add_option(Option::Type::NoArg, short_key, long_key, description, [callback = std::move(callback)](Slice) { + callback(); + return Status::OK(); + }); +} + +void OptionParser::add_check(std::function check) { + checks_.push_back(std::move(check)); +} + +Result> OptionParser::run(int argc, char *argv[], int expected_non_option_count) { +#if TD_PORT_WINDOWS +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + LPWSTR *utf16_argv = CommandLineToArgvW(GetCommandLineW(), &argc); + if (utf16_argv == nullptr) { + return Status::Error("Failed to parse command line"); + } + vector args_storage(argc); + vector args(argc); + for (int i = 0; i < argc; i++) { + TRY_RESULT_ASSIGN(args_storage[i], from_wstring(utf16_argv[i])); + args[i] = &args_storage[i][0]; + } + LocalFree(utf16_argv); + argv = &args[0]; +#endif +#endif + + return run_impl(argc, argv, expected_non_option_count); +} + +Result> OptionParser::run_impl(int argc, char *argv[], int expected_non_option_count) { + FlatHashMap short_options; + FlatHashMap long_options; + for (auto &opt : options_) { + if (opt.short_key != '\0') { + short_options[opt.short_key] = &opt; + } + if (!opt.long_key.empty()) { + long_options[opt.long_key] = &opt; + } + } + + vector non_options; + for (int arg_pos = 1; arg_pos < argc; arg_pos++) { + const char *arg = argv[arg_pos]; + if (arg[0] != '-' || arg[1] == '\0') { + non_options.push_back(argv[arg_pos]); + continue; + } + if (arg[1] == '-' && arg[2] == '\0') { + // "--"; after it everything is non-option + while (++arg_pos < argc) { + non_options.push_back(argv[arg_pos]); + } + break; + } + + if (arg[1] == '-') { + // long option + Slice long_arg(arg + 2); + Slice parameter; + auto equal_pos = long_arg.find('='); + bool has_equal = equal_pos != Slice::npos; + if (has_equal) { + parameter = long_arg.substr(equal_pos + 1); + long_arg = long_arg.substr(0, equal_pos); + } + + auto it = long_options.find(long_arg.str()); + if (it == long_options.end()) { + return Status::Error(PSLICE() << "Option \"" << long_arg << "\" is unrecognized"); + } + + auto option = it->second; + switch (option->type) { + case Option::Type::NoArg: + if (has_equal) { + return Status::Error(PSLICE() << "Option \"" << long_arg << "\" must not have an argument"); + } + break; + case Option::Type::Arg: + if (!has_equal) { + if (++arg_pos == argc) { + return Status::Error(PSLICE() << "Option \"" << long_arg << "\" requires an argument"); + } + parameter = Slice(argv[arg_pos]); + } + break; + default: + UNREACHABLE(); + } + + TRY_STATUS(option->arg_callback(parameter)); + continue; + } + + for (size_t opt_pos = 1; arg[opt_pos] != '\0'; opt_pos++) { + auto it = short_options.find(arg[opt_pos]); + if (it == short_options.end()) { + return Status::Error(PSLICE() << "Option \"" << arg[opt_pos] << "\" is unrecognized"); + } + + auto option = it->second; + Slice parameter; + switch (option->type) { + case Option::Type::NoArg: + // nothing to do + break; + case Option::Type::Arg: + if (arg[opt_pos + 1] == '\0') { + if (++arg_pos == argc) { + return Status::Error(PSLICE() << "Option \"" << arg[opt_pos] << "\" requires an argument"); + } + parameter = Slice(argv[arg_pos]); + } else { + parameter = Slice(arg + opt_pos + 1); + opt_pos += parameter.size(); + } + break; + default: + UNREACHABLE(); + } + + TRY_STATUS(option->arg_callback(parameter)); + } + } + if (expected_non_option_count >= 0 && non_options.size() != static_cast(expected_non_option_count)) { + if (expected_non_option_count == 0) { + return Status::Error("Unexpected non-option parameters specified"); + } + if (non_options.size() > static_cast(expected_non_option_count)) { + return Status::Error("Too many non-option parameters specified"); + } else { + return Status::Error("Too few non-option parameters specified"); + } + } + for (auto &check : checks_) { + TRY_STATUS(check()); + } + + return std::move(non_options); +} + +StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o) { + if (!o.usage_.empty()) { + sb << "Usage: " << o.usage_ << "\n\n"; + } + if (!o.description_.empty()) { + sb << o.description_ << ". "; + } + sb << "Options:\n"; + + size_t max_length = 0; + for (auto &opt : o.options_) { + size_t length = 2; + if (!opt.long_key.empty()) { + length += 4 + opt.long_key.size(); + } + if (opt.type != OptionParser::Option::Type::NoArg) { + length += 6; + } + if (length > max_length) { + max_length = length; + } + } + max_length++; + + for (auto &opt : o.options_) { + bool has_short_key = opt.short_key != '\0'; + sb << " "; + size_t length = max_length; + if (has_short_key) { + sb << '-' << opt.short_key; + } else { + sb << " "; + } + length -= 2; + if (!opt.long_key.empty()) { + if (has_short_key) { + sb << ", "; + } else { + sb << " "; + } + sb << "--" << opt.long_key; + length -= 4 + opt.long_key.size(); + } + if (opt.type != OptionParser::Option::Type::NoArg) { + sb << "="; + length -= 6; + } + sb << string(length, ' ') << opt.description; + sb << '\n'; + } + return sb; +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.h new file mode 100644 index 0000000000..82d4296dca --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionParser.h @@ -0,0 +1,81 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include + +namespace td { + +class OptionParser { + class Option { + public: + enum class Type { NoArg, Arg }; + Type type; + char short_key; + string long_key; + string description; + std::function arg_callback; + }; + + void add_option(Option::Type type, char short_key, Slice long_key, Slice description, + std::function callback); + + public: + template + static std::function parse_integer(T &value) { + return [&value](Slice value_str) { + TRY_RESULT_ASSIGN(value, to_integer_safe(value_str)); + return Status::OK(); + }; + } + + static std::function parse_string(string &value) { + return [&value](Slice value_str) { + value = value_str.str(); + }; + } + + void set_usage(Slice executable_name, Slice usage); + + void set_description(string description); + + void add_checked_option(char short_key, Slice long_key, Slice description, std::function callback); + + void add_checked_option(char short_key, Slice long_key, Slice description, std::function callback); + + void add_option(char short_key, Slice long_key, Slice description, std::function callback) = delete; + + void add_option(char short_key, Slice long_key, Slice description, std::function callback) = delete; + + void add_option(char short_key, Slice long_key, Slice description, std::function callback); + + void add_option(char short_key, Slice long_key, Slice description, std::function callback); + + void add_check(std::function check); + + // returns found non-option parameters + Result> run(int argc, char *argv[], int expected_non_option_count = -1) TD_WARN_UNUSED_RESULT; + + // for testing only + Result> run_impl(int argc, char *argv[], int expected_non_option_count) TD_WARN_UNUSED_RESULT; + + friend StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o); + + private: + vector