From e1ec72eab6d00b3ba38e5932bc88920f103b6e4a Mon Sep 17 00:00:00 2001 From: aunsane Date: Fri, 27 Apr 2018 21:33:17 +0300 Subject: Telegram: initial commit - tdlib moved to telegram dir --- protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt | 244 +++++ .../tdlib/td/tdutils/generate/CMakeLists.txt | 64 ++ .../tdutils/generate/generate_mime_types_gperf.cpp | 146 +++ .../tdlib/td/tdutils/generate/mime_types.txt | 765 ++++++++++++++ .../tdlib/td/tdutils/td/utils/AesCtrByteFlow.h | 55 + .../Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp | 251 +++++ .../Telegram/tdlib/td/tdutils/td/utils/BigNum.h | 108 ++ .../tdlib/td/tdutils/td/utils/BufferedFd.h | 199 ++++ .../tdlib/td/tdutils/td/utils/BufferedReader.h | 61 ++ .../Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h | 288 +++++ .../tdlib/td/tdutils/td/utils/ChangesProcessor.h | 61 ++ .../Telegram/tdlib/td/tdutils/td/utils/Closure.h | 169 +++ .../Telegram/tdlib/td/tdutils/td/utils/Container.h | 149 +++ .../tdlib/td/tdutils/td/utils/Enumerator.h | 45 + .../Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp | 92 ++ .../Telegram/tdlib/td/tdutils/td/utils/FileLog.h | 37 + .../tdlib/td/tdutils/td/utils/FloodControlFast.h | 62 ++ .../tdlib/td/tdutils/td/utils/FloodControlStrict.h | 97 ++ .../Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp | 20 + .../Telegram/tdlib/td/tdutils/td/utils/GitInfo.h | 19 + .../Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp | 191 ++++ .../Telegram/tdlib/td/tdutils/td/utils/Gzip.h | 104 ++ .../tdlib/td/tdutils/td/utils/GzipByteFlow.cpp | 70 ++ .../tdlib/td/tdutils/td/utils/GzipByteFlow.h | 48 + .../tdlib/td/tdutils/td/utils/HazardPointers.h | 133 +++ .../Telegram/tdlib/td/tdutils/td/utils/Heap.h | 152 +++ .../Telegram/tdlib/td/tdutils/td/utils/Hints.cpp | 191 ++++ .../Telegram/tdlib/td/tdutils/td/utils/Hints.h | 76 ++ .../Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp | 189 ++++ .../Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h | 39 + .../tdlib/td/tdutils/td/utils/JsonBuilder.cpp | 648 ++++++++++++ .../tdlib/td/tdutils/td/utils/JsonBuilder.h | 760 ++++++++++++++ .../Telegram/tdlib/td/tdutils/td/utils/List.h | 92 ++ .../Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h | 83 ++ .../tdlib/td/tdutils/td/utils/MimeType.cpp | 44 + .../Telegram/tdlib/td/tdutils/td/utils/MimeType.h | 20 + .../tdlib/td/tdutils/td/utils/MovableValue.h | 40 + .../Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h | 449 ++++++++ .../tdlib/td/tdutils/td/utils/MpmcWaiter.h | 106 ++ .../tdlib/td/tdutils/td/utils/MpscLinkQueue.h | 174 +++ .../tdlib/td/tdutils/td/utils/MpscPollableQueue.h | 154 +++ .../Telegram/tdlib/td/tdutils/td/utils/Named.h | 27 + .../tdlib/td/tdutils/td/utils/ObjectPool.h | 249 +++++ .../Telegram/tdlib/td/tdutils/td/utils/Observer.h | 41 + .../tdlib/td/tdutils/td/utils/OptionsParser.h | 150 +++ .../td/tdutils/td/utils/OrderedEventsProcessor.h | 87 ++ .../Telegram/tdlib/td/tdutils/td/utils/Parser.h | 183 ++++ .../Telegram/tdlib/td/tdutils/td/utils/PathView.h | 116 ++ .../Telegram/tdlib/td/tdutils/td/utils/Random.cpp | 108 ++ .../Telegram/tdlib/td/tdutils/td/utils/Random.h | 30 + .../tdlib/td/tdutils/td/utils/ScopeGuard.h | 76 ++ .../tdlib/td/tdutils/td/utils/SharedObjectPool.h | 276 +++++ .../tdlib/td/tdutils/td/utils/Slice-decl.h | 187 ++++ .../Telegram/tdlib/td/tdutils/td/utils/Slice.h | 275 +++++ .../Telegram/tdlib/td/tdutils/td/utils/SpinLock.h | 58 + .../tdlib/td/tdutils/td/utils/StackAllocator.cpp | 18 + .../tdlib/td/tdutils/td/utils/StackAllocator.h | 82 ++ .../Telegram/tdlib/td/tdutils/td/utils/Status.cpp | 54 + .../Telegram/tdlib/td/tdutils/td/utils/Status.h | 458 ++++++++ .../Telegram/tdlib/td/tdutils/td/utils/Storer.h | 86 ++ .../tdlib/td/tdutils/td/utils/StorerBase.h | 25 + .../tdlib/td/tdutils/td/utils/StringBuilder.cpp | 102 ++ .../tdlib/td/tdutils/td/utils/StringBuilder.h | 138 +++ .../Telegram/tdlib/td/tdutils/td/utils/Time.cpp | 19 + .../Telegram/tdlib/td/tdutils/td/utils/Time.h | 104 ++ .../Telegram/tdlib/td/tdutils/td/utils/TimedStat.h | 71 ++ .../Telegram/tdlib/td/tdutils/td/utils/Timer.cpp | 41 + .../Telegram/tdlib/td/tdutils/td/utils/Timer.h | 38 + .../Telegram/tdlib/td/tdutils/td/utils/Variant.h | 286 +++++ .../Telegram/tdlib/td/tdutils/td/utils/base64.cpp | 261 +++++ .../Telegram/tdlib/td/tdutils/td/utils/base64.h | 26 + .../Telegram/tdlib/td/tdutils/td/utils/benchmark.h | 132 +++ .../Telegram/tdlib/td/tdutils/td/utils/buffer.cpp | 105 ++ .../Telegram/tdlib/td/tdutils/td/utils/buffer.h | 708 +++++++++++++ .../Telegram/tdlib/td/tdutils/td/utils/common.h | 126 +++ .../Telegram/tdlib/td/tdutils/td/utils/config.h | 3 + .../Telegram/tdlib/td/tdutils/td/utils/config.h.in | 3 + .../Telegram/tdlib/td/tdutils/td/utils/crypto.cpp | 541 ++++++++++ .../Telegram/tdlib/td/tdutils/td/utils/crypto.h | 79 ++ .../tdlib/td/tdutils/td/utils/filesystem.cpp | 123 +++ .../tdlib/td/tdutils/td/utils/filesystem.h | 22 + .../tdlib/td/tdutils/td/utils/find_boundary.cpp | 53 + .../tdlib/td/tdutils/td/utils/find_boundary.h | 17 + .../Telegram/tdlib/td/tdutils/td/utils/format.h | 312 ++++++ .../Telegram/tdlib/td/tdutils/td/utils/int_types.h | 65 ++ .../Telegram/tdlib/td/tdutils/td/utils/invoke.h | 178 ++++ .../Telegram/tdlib/td/tdutils/td/utils/logging.cpp | 238 +++++ .../Telegram/tdlib/td/tdutils/td/utils/logging.h | 279 +++++ .../Telegram/tdlib/td/tdutils/td/utils/misc.cpp | 78 ++ .../Telegram/tdlib/td/tdutils/td/utils/misc.h | 337 ++++++ .../Telegram/tdlib/td/tdutils/td/utils/optional.h | 36 + .../tdlib/td/tdutils/td/utils/overloaded.h | 39 + .../tdlib/td/tdutils/td/utils/port/Clocks.cpp | 23 + .../tdlib/td/tdutils/td/utils/port/Clocks.h | 28 + .../tdlib/td/tdutils/td/utils/port/CxCli.h | 133 +++ .../tdlib/td/tdutils/td/utils/port/EventFd.h | 33 + .../tdlib/td/tdutils/td/utils/port/EventFdBase.h | 32 + .../Telegram/tdlib/td/tdutils/td/utils/port/Fd.cpp | 1104 ++++++++++++++++++++ .../Telegram/tdlib/td/tdutils/td/utils/port/Fd.h | 226 ++++ .../tdlib/td/tdutils/td/utils/port/FileFd.cpp | 481 +++++++++ .../tdlib/td/tdutils/td/utils/port/FileFd.h | 63 ++ .../tdlib/td/tdutils/td/utils/port/IPAddress.cpp | 361 +++++++ .../tdlib/td/tdutils/td/utils/port/IPAddress.h | 71 ++ .../Telegram/tdlib/td/tdutils/td/utils/port/Poll.h | 35 + .../tdlib/td/tdutils/td/utils/port/PollBase.h | 27 + .../tdlib/td/tdutils/td/utils/port/RwMutex.h | 147 +++ .../td/tdutils/td/utils/port/ServerSocketFd.cpp | 160 +++ .../td/tdutils/td/utils/port/ServerSocketFd.h | 43 + .../tdlib/td/tdutils/td/utils/port/SocketFd.cpp | 139 +++ .../tdlib/td/tdutils/td/utils/port/SocketFd.h | 57 + .../tdlib/td/tdutils/td/utils/port/Stat.cpp | 337 ++++++ .../Telegram/tdlib/td/tdutils/td/utils/port/Stat.h | 53 + .../tdlib/td/tdutils/td/utils/port/config.h | 46 + .../td/tdutils/td/utils/port/detail/Epoll.cpp | 114 ++ .../tdlib/td/tdutils/td/utils/port/detail/Epoll.h | 51 + .../td/tdutils/td/utils/port/detail/EventFdBsd.cpp | 93 ++ .../td/tdutils/td/utils/port/detail/EventFdBsd.h | 47 + .../tdutils/td/utils/port/detail/EventFdLinux.cpp | 74 ++ .../td/tdutils/td/utils/port/detail/EventFdLinux.h | 44 + .../td/utils/port/detail/EventFdWindows.cpp | 51 + .../tdutils/td/utils/port/detail/EventFdWindows.h | 46 + .../td/tdutils/td/utils/port/detail/KQueue.cpp | 160 +++ .../tdlib/td/tdutils/td/utils/port/detail/KQueue.h | 62 ++ .../tdlib/td/tdutils/td/utils/port/detail/Poll.cpp | 92 ++ .../tdlib/td/tdutils/td/utils/port/detail/Poll.h | 50 + .../td/tdutils/td/utils/port/detail/Select.cpp | 119 +++ .../tdlib/td/tdutils/td/utils/port/detail/Select.h | 59 ++ .../tdutils/td/utils/port/detail/ThreadIdGuard.cpp | 52 + .../tdutils/td/utils/port/detail/ThreadIdGuard.h | 26 + .../tdutils/td/utils/port/detail/ThreadPthread.h | 90 ++ .../td/tdutils/td/utils/port/detail/ThreadStl.h | 64 ++ .../tdutils/td/utils/port/detail/WineventPoll.cpp | 97 ++ .../td/tdutils/td/utils/port/detail/WineventPoll.h | 52 + .../tdlib/td/tdutils/td/utils/port/path.cpp | 383 +++++++ .../Telegram/tdlib/td/tdutils/td/utils/port/path.h | 225 ++++ .../tdlib/td/tdutils/td/utils/port/platform.h | 106 ++ .../tdlib/td/tdutils/td/utils/port/signals.cpp | 298 ++++++ .../tdlib/td/tdutils/td/utils/port/signals.h | 34 + .../tdlib/td/tdutils/td/utils/port/sleep.cpp | 37 + .../tdlib/td/tdutils/td/utils/port/sleep.h | 15 + .../tdlib/td/tdutils/td/utils/port/thread.h | 34 + .../td/tdutils/td/utils/port/thread_local.cpp | 41 + .../tdlib/td/tdutils/td/utils/port/thread_local.h | 69 ++ .../td/tdutils/td/utils/port/wstring_convert.cpp | 63 ++ .../td/tdutils/td/utils/port/wstring_convert.h | 31 + .../Telegram/tdlib/td/tdutils/td/utils/queue.h | 484 +++++++++ .../Telegram/tdlib/td/tdutils/td/utils/tests.h | 205 ++++ .../tdlib/td/tdutils/td/utils/tl_helpers.h | 203 ++++ .../tdlib/td/tdutils/td/utils/tl_parsers.cpp | 29 + .../tdlib/td/tdutils/td/utils/tl_parsers.h | 242 +++++ .../tdlib/td/tdutils/td/utils/tl_storers.h | 281 +++++ .../tdlib/td/tdutils/td/utils/type_traits.h | 22 + .../Telegram/tdlib/td/tdutils/td/utils/unicode.cpp | 574 ++++++++++ .../Telegram/tdlib/td/tdutils/td/utils/unicode.h | 28 + .../Telegram/tdlib/td/tdutils/td/utils/utf8.cpp | 124 +++ .../Telegram/tdlib/td/tdutils/td/utils/utf8.h | 106 ++ .../Telegram/tdlib/td/tdutils/test/Enumerator.cpp | 24 + .../tdlib/td/tdutils/test/HazardPointers.cpp | 58 + .../Telegram/tdlib/td/tdutils/test/MpmcQueue.cpp | 205 ++++ .../Telegram/tdlib/td/tdutils/test/MpmcWaiter.cpp | 117 +++ .../tdlib/td/tdutils/test/MpscLinkQueue.cpp | 115 ++ .../td/tdutils/test/OrderedEventsProcessor.cpp | 36 + .../tdlib/td/tdutils/test/SharedObjectPool.cpp | 96 ++ .../Telegram/tdlib/td/tdutils/test/crypto.cpp | 166 +++ .../Telegram/tdlib/td/tdutils/test/filesystem.cpp | 41 + protocols/Telegram/tdlib/td/tdutils/test/gzip.cpp | 113 ++ protocols/Telegram/tdlib/td/tdutils/test/heap.cpp | 178 ++++ protocols/Telegram/tdlib/td/tdutils/test/json.cpp | 94 ++ protocols/Telegram/tdlib/td/tdutils/test/misc.cpp | 262 +++++ protocols/Telegram/tdlib/td/tdutils/test/pq.cpp | 118 +++ .../Telegram/tdlib/td/tdutils/test/variant.cpp | 75 ++ 171 files changed, 24215 insertions(+) create mode 100644 protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt create mode 100644 protocols/Telegram/tdlib/td/tdutils/generate/CMakeLists.txt create mode 100644 protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/List.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/OptionsParser.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/OrderedEventsProcessor.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Parser.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/PathView.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Random.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Random.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/ScopeGuard.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/SharedObjectPool.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Slice-decl.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Slice.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/SpinLock.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/StackAllocator.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/StackAllocator.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Status.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Status.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Storer.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/StorerBase.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/StringBuilder.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/StringBuilder.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Time.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Time.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/TimedStat.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Timer.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Timer.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/Variant.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/base64.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/base64.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/benchmark.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/buffer.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/buffer.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/common.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/config.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/config.h.in create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/crypto.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/crypto.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/filesystem.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/filesystem.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/find_boundary.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/find_boundary.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/format.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/int_types.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/invoke.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/logging.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/logging.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/misc.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/misc.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/optional.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/overloaded.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Clocks.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Clocks.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/CxCli.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/EventFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/EventFdBase.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Fd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Fd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/FileFd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/FileFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/IPAddress.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/IPAddress.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Poll.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/PollBase.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/RwMutex.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/ServerSocketFd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/ServerSocketFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/SocketFd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/SocketFd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Stat.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/Stat.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/config.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Epoll.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Epoll.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/KQueue.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/KQueue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Poll.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Poll.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Select.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/Select.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/ThreadPthread.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/ThreadStl.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/path.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/path.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/platform.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/signals.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/signals.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/sleep.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/sleep.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/thread.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/thread_local.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/thread_local.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/wstring_convert.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/port/wstring_convert.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/queue.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/tests.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/tl_helpers.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/tl_parsers.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/tl_parsers.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/tl_storers.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/type_traits.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/unicode.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/unicode.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/utf8.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/td/utils/utf8.h create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/Enumerator.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/HazardPointers.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/MpmcQueue.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/MpmcWaiter.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/MpscLinkQueue.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/OrderedEventsProcessor.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/SharedObjectPool.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/crypto.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/filesystem.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/gzip.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/heap.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/json.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/misc.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/pq.cpp create mode 100644 protocols/Telegram/tdlib/td/tdutils/test/variant.cpp (limited to 'protocols/Telegram/tdlib/td/tdutils') diff --git a/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt b/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt new file mode 100644 index 0000000000..1fbc34df32 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/CMakeLists.txt @@ -0,0 +1,244 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +if (NOT ZLIB_FOUND) + find_package(ZLIB) +endif() +if (ZLIB_FOUND) + set(TD_HAVE_ZLIB 1) + message(STATUS "Found ZLIB: ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}") + + # OpenSSL internally depends on zlib + if (NOT OPENSSL_FOUND) + find_package(OpenSSL) + endif() + if (OPENSSL_FOUND) + set(TD_HAVE_OPENSSL 1) + endif() +endif() + +configure_file(td/utils/config.h.in td/utils/config.h @ONLY) + +add_subdirectory(generate) + +# TDUTILS +set_source_files_properties(${TDMIME_AUTO} PROPERTIES GENERATED TRUE) +if (CLANG OR GCC) + set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-conversion") +elseif (MSVC) + set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4267") +endif() +if (CLANG) + set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-deprecated-register") +endif() + +set(TDUTILS_SOURCE + td/utils/port/Clocks.cpp + td/utils/port/Fd.cpp + td/utils/port/FileFd.cpp + td/utils/port/IPAddress.cpp + td/utils/port/path.cpp + td/utils/port/ServerSocketFd.cpp + td/utils/port/signals.cpp + td/utils/port/sleep.cpp + td/utils/port/SocketFd.cpp + td/utils/port/Stat.cpp + td/utils/port/thread_local.cpp + td/utils/port/wstring_convert.cpp + + td/utils/port/detail/Epoll.cpp + td/utils/port/detail/EventFdBsd.cpp + td/utils/port/detail/EventFdLinux.cpp + td/utils/port/detail/EventFdWindows.cpp + td/utils/port/detail/KQueue.cpp + td/utils/port/detail/Poll.cpp + td/utils/port/detail/Select.cpp + td/utils/port/detail/ThreadIdGuard.cpp + td/utils/port/detail/WineventPoll.cpp + + ${TDMIME_AUTO} + + td/utils/base64.cpp + td/utils/BigNum.cpp + td/utils/buffer.cpp + td/utils/crypto.cpp + td/utils/FileLog.cpp + td/utils/filesystem.cpp + td/utils/find_boundary.cpp + td/utils/Gzip.cpp + td/utils/GzipByteFlow.cpp + td/utils/Hints.cpp + td/utils/HttpUrl.cpp + td/utils/JsonBuilder.cpp + td/utils/logging.cpp + td/utils/misc.cpp + td/utils/MimeType.cpp + td/utils/Random.cpp + td/utils/StackAllocator.cpp + td/utils/Status.cpp + td/utils/StringBuilder.cpp + td/utils/Time.cpp + td/utils/Timer.cpp + td/utils/tl_parsers.cpp + td/utils/unicode.cpp + td/utils/utf8.cpp + + td/utils/port/Clocks.h + td/utils/port/config.h + td/utils/port/CxCli.h + td/utils/port/EventFd.h + td/utils/port/EventFdBase.h + td/utils/port/Fd.h + td/utils/port/FileFd.h + td/utils/port/IPAddress.h + td/utils/port/path.h + td/utils/port/platform.h + td/utils/port/Poll.h + td/utils/port/PollBase.h + td/utils/port/RwMutex.h + td/utils/port/ServerSocketFd.h + td/utils/port/signals.h + td/utils/port/sleep.h + td/utils/port/SocketFd.h + td/utils/port/Stat.h + td/utils/port/thread.h + td/utils/port/thread_local.h + td/utils/port/wstring_convert.h + + td/utils/port/detail/Epoll.h + td/utils/port/detail/EventFdBsd.h + td/utils/port/detail/EventFdLinux.h + td/utils/port/detail/EventFdWindows.h + td/utils/port/detail/KQueue.h + td/utils/port/detail/Poll.h + td/utils/port/detail/Select.h + td/utils/port/detail/ThreadIdGuard.h + td/utils/port/detail/ThreadPthread.h + td/utils/port/detail/ThreadStl.h + td/utils/port/detail/WineventPoll.h + + td/utils/AesCtrByteFlow.h + td/utils/base64.h + td/utils/benchmark.h + td/utils/BigNum.h + td/utils/buffer.h + td/utils/BufferedFd.h + td/utils/BufferedReader.h + td/utils/ByteFlow.h + td/utils/ChangesProcessor.h + td/utils/Closure.h + td/utils/common.h + td/utils/Container.h + td/utils/crypto.h + td/utils/Enumerator.h + td/utils/FileLog.h + td/utils/filesystem.h + td/utils/find_boundary.h + td/utils/FloodControlFast.h + td/utils/FloodControlStrict.h + td/utils/format.h + td/utils/Gzip.h + td/utils/GzipByteFlow.h + td/utils/HazardPointers.h + td/utils/Heap.h + td/utils/Hints.h + td/utils/HttpUrl.h + td/utils/int_types.h + td/utils/invoke.h + td/utils/JsonBuilder.h + td/utils/List.h + td/utils/logging.h + td/utils/MemoryLog.h + td/utils/MimeType.h + td/utils/misc.h + td/utils/MovableValue.h + td/utils/MpmcQueue.h + td/utils/MpmcWaiter.h + td/utils/MpscPollableQueue.h + td/utils/MpscLinkQueue.h + td/utils/Named.h + td/utils/ObjectPool.h + td/utils/Observer.h + td/utils/optional.h + td/utils/OptionsParser.h + td/utils/OrderedEventsProcessor.h + td/utils/overloaded.h + td/utils/Parser.h + td/utils/PathView.h + td/utils/queue.h + td/utils/Random.h + td/utils/ScopeGuard.h + td/utils/SharedObjectPool.h + td/utils/Slice-decl.h + td/utils/Slice.h + td/utils/SpinLock.h + td/utils/StackAllocator.h + td/utils/Status.h + td/utils/Storer.h + td/utils/StorerBase.h + td/utils/StringBuilder.h + td/utils/tests.h + td/utils/Time.h + td/utils/TimedStat.h + td/utils/Timer.h + td/utils/tl_helpers.h + td/utils/tl_parsers.h + td/utils/tl_storers.h + td/utils/type_traits.h + td/utils/unicode.h + td/utils/utf8.h + td/utils/Variant.h +) + +set(TDUTILS_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp + PARENT_SCOPE +) + +#RULES +#LIBRARIES +add_library(tdutils STATIC ${TDUTILS_SOURCE}) +if (WIN32) + # find_library(WS2_32_LIBRARY ws2_32) + # find_library(MSWSOCK_LIBRARY Mswsock) + # target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY}) + target_link_libraries(tdutils PRIVATE ws2_32 Mswsock) +endif() +if (NOT CMAKE_CROSSCOMPILING) + add_dependencies(tdutils tdmime_auto) +endif() + +if (DEFINED CMAKE_THREAD_LIBS_INIT) + target_link_libraries(tdutils PUBLIC ${CMAKE_THREAD_LIBS_INIT}) +endif() +target_include_directories(tdutils PUBLIC $ $) + +if (OPENSSL_FOUND) + target_link_libraries(tdutils PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) + target_include_directories(tdutils SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) +endif() + +if (ZLIB_FOUND) + target_link_libraries(tdutils PRIVATE ${ZLIB_LIBRARIES}) + target_include_directories(tdutils SYSTEM PRIVATE ${ZLIB_INCLUDE_DIR}) +endif() + +install(TARGETS tdutils EXPORT TdTargets + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + RUNTIME DESTINATION bin + INCLUDES DESTINATION include +) diff --git a/protocols/Telegram/tdlib/td/tdutils/generate/CMakeLists.txt b/protocols/Telegram/tdlib/td/tdutils/generate/CMakeLists.txt new file mode 100644 index 0000000000..69697b04b8 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/generate/CMakeLists.txt @@ -0,0 +1,64 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +# Generates files for MIME type <-> extension conversions +# DEPENDS ON: gperf grep bash/powershell + +file(MAKE_DIRECTORY auto) + +set(TDMIME_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp +) +set(TDMIME_AUTO + ${TDMIME_SOURCE} + PARENT_SCOPE +) + +add_custom_target(tdmime_auto DEPENDS ${TDMIME_SOURCE}) + +if (NOT CMAKE_CROSSCOMPILING) + find_program(GPERF_EXECUTABLE gperf) + if (NOT GPERF_EXECUTABLE) + message(FATAL_ERROR "Could NOT find gperf. Path to gperf needs to be specified manually, i.e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"\" .')") + endif() + + set(GPERF_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.gperf + ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.gperf + ) + + set(GPERF_GEN_SOURCE generate_mime_types_gperf.cpp) + + add_executable(generate_mime_types_gperf ${GPERF_GEN_SOURCE}) + + add_custom_command( + OUTPUT ${GPERF_FILES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND generate_mime_types_gperf mime_types.txt ${GPERF_FILES} + DEPENDS generate_mime_types_gperf mime_types.txt + ) + + if (WIN32) + set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/mime_type_to_extension.cpp auto/mime_type_to_extension.gperf) + else() + set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 auto/mime_type_to_extension.gperf | grep -v __gnu_inline__ > auto/mime_type_to_extension.cpp) + endif() + add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${MIME_TYPE_TO_EXTENSION_CMD} + DEPENDS auto/mime_type_to_extension.gperf + ) + + if (WIN32) + set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/extension_to_mime_type.cpp auto/extension_to_mime_type.gperf) + else() + set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 auto/extension_to_mime_type.gperf | grep -v __gnu_inline__ > auto/extension_to_mime_type.cpp) + endif() + add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${EXTENSION_TO_MIME_TYPE_CMD} + DEPENDS auto/extension_to_mime_type.gperf + ) +endif() diff --git a/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp b/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp new file mode 100644 index 0000000000..ac7ff68605 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp @@ -0,0 +1,146 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +std::pair split(std::string s, char delimiter = ' ') { + auto delimiter_pos = s.find(delimiter); + if (delimiter_pos == std::string::npos) { + return {std::move(s), ""}; + } else { + auto head = s.substr(0, delimiter_pos); + auto tail = s.substr(delimiter_pos + 1); + return {head, tail}; + } +} + +bool generate(const char *file_name, const char *from_name, const char *to_name, + const std::map &map) { + // binary mode is needed for MSYS2 gperf + std::ofstream out(file_name, std::ios_base::trunc | std::ios_base::binary); + if (!out) { + std::cerr << "Can't open output file \"" << file_name << std::endl; + return false; + } + + out << "%struct-type\n"; + out << "%ignore-case\n"; + out << "%language=ANSI-C\n"; + out << "%readonly-tables\n"; + out << "%includes\n"; + out << "%enum\n"; + out << "%define slot-name " << from_name << "\n"; + out << "%define initializer-suffix ,nullptr\n"; + out << "%define slot-name " << from_name << "\n"; + out << "%define hash-function-name " << from_name << "_hash\n"; + out << "%define lookup-function-name search_" << from_name << "\n"; + // out << "%define class-name " << from_name << "_to_" << to_name << "\n"; + out << "struct " << from_name << "_and_" << to_name << " {\n"; + out << " const char *" << from_name << ";\n"; + out << " const char *" << to_name << ";\n"; + out << "}\n"; + out << "%%\n"; + + for (auto &value : map) { + out << '"' << value.first << "\", \"" << value.second << '"' << "\n"; + } + + out << "%%\n"; + out << "const char *" << from_name << "_to_" << to_name << "(const char *" << from_name << ", size_t " << from_name + << "_len) {\n"; + out << " const auto &result = search_" << from_name << "(" << from_name << ", " << from_name << "_len);\n"; + out << " if (result == nullptr) {\n"; + out << " return nullptr;\n"; + out << " }\n"; + out << "\n"; + out << " return result->" << to_name << ";\n"; + out << "}\n"; + + return true; +} + +int main(int argc, char *argv[]) { + if (argc != 4) { + std::cerr << "Wrong number of arguments supplied. Expected 'generate_mime_types_gperf " + " '" + << std::endl; + return EXIT_FAILURE; + } + + std::ifstream mime_types_file(argv[1]); + if (!mime_types_file) { + std::cerr << "Can't open input file \"" << argv[1] << std::endl; + return EXIT_FAILURE; + } + + std::map mime_type_to_extension; + std::map extension_to_mime_type; + + std::string line; + while (std::getline(mime_types_file, line)) { + while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) { + line.pop_back(); + } + + std::string mime_type; + std::string extensions_string; + std::tie(mime_type, extensions_string) = split(line, '\t'); + + if (mime_type.empty()) { + std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl; + continue; + } + + auto extensions_start_position = extensions_string.find_first_not_of(" \t"); + if (extensions_start_position == std::string::npos) { + std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl; + continue; + } + extensions_string = extensions_string.substr(extensions_start_position); + + std::vector extensions; + while (!extensions_string.empty()) { + extensions.push_back(""); + std::tie(extensions.back(), extensions_string) = split(extensions_string); + } + assert(!extensions.empty()); + + std::map preffered_extensions{{"image/jpeg", "jpg"}, {"audio/mpeg", "mp3"}, + {"audio/midi", "midi"}, {"text/x-pascal", "pas"}, + {"text/x-asm", "asm"}, {"video/quicktime", "mov"}}; + std::size_t index = 0; + if (preffered_extensions.count(mime_type) != 0) { + index = std::find(extensions.begin(), extensions.end(), preffered_extensions[mime_type]) - extensions.begin(); + assert(index < extensions.size()); + } + if (mime_type_to_extension.emplace_hint(mime_type_to_extension.end(), mime_type, extensions[index])->second != + extensions[index]) { + std::cerr << "MIME type \"" << mime_type << "\" has more than one extensions list" << std::endl; + } + + for (auto &extension : extensions) { + if (!extension_to_mime_type.emplace(extension, mime_type).second) { + std::cerr << "Extension \"" << extension << "\" matches more than one type" << std::endl; + } + } + } + + if (!generate(argv[2], "mime_type", "extension", mime_type_to_extension)) { + return EXIT_FAILURE; + } + if (!generate(argv[3], "extension", "mime_type", extension_to_mime_type)) { + return EXIT_FAILURE; + } +} diff --git a/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt b/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt new file mode 100644 index 0000000000..a2d4abbf29 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/generate/mime_types.txt @@ -0,0 +1,765 @@ +application/andrew-inset ez +application/applixware aw +application/atom+xml atom +application/atomcat+xml atomcat +application/atomsvc+xml atomsvc +application/ccxml+xml ccxml +application/cdmi-capability cdmia +application/cdmi-container cdmic +application/cdmi-domain cdmid +application/cdmi-object cdmio +application/cdmi-queue cdmiq +application/cu-seeme cu +application/davmount+xml davmount +application/docbook+xml dbk +application/dssc+der dssc +application/dssc+xml xdssc +application/ecmascript ecma +application/emma+xml emma +application/epub+zip epub +application/exi exi +application/font-tdpfr pfr +application/gml+xml gml +application/gpx+xml gpx +application/gxf gxf +application/hyperstudio stk +application/inkml+xml ink inkml +application/ipfix ipfix +application/java-archive jar +application/java-serialized-object ser +application/java-vm class +application/javascript js +application/json json +application/jsonml+json jsonml +application/lost+xml lostxml +application/mac-binhex40 hqx +application/mac-compactpro cpt +application/mads+xml mads +application/marc mrc +application/marcxml+xml mrcx +application/mathematica ma nb mb +application/mathml+xml mathml +application/mbox mbox +application/mediaservercontrol+xml mscml +application/metalink+xml metalink +application/metalink4+xml meta4 +application/mets+xml mets +application/mods+xml mods +application/mp21 m21 mp21 +application/mp4 mp4s +application/msword doc dot +application/mxf mxf +application/octet-stream bin dms lrf mar so dist distz pkg bpk dump elc deploy +application/oda oda +application/oebps-package+xml opf +application/ogg ogx +application/omdoc+xml omdoc +application/onenote onetoc onetoc2 onetmp onepkg +application/oxps oxps +application/patch-ops-error+xml xer +application/pdf pdf +application/pgp-encrypted pgp +application/pgp-signature asc sig +application/pics-rules prf +application/pkcs10 p10 +application/pkcs7-mime p7m p7c +application/pkcs7-signature p7s +application/pkcs8 p8 +application/pkix-attr-cert ac +application/pkix-cert cer +application/pkix-crl crl +application/pkix-pkipath pkipath +application/pkixcmp pki +application/pls+xml pls +application/postscript ai eps ps +application/prs.cww cww +application/pskc+xml pskcxml +application/rdf+xml rdf +application/reginfo+xml rif +application/relax-ng-compact-syntax rnc +application/resource-lists+xml rl +application/resource-lists-diff+xml rld +application/rls-services+xml rs +application/rpki-ghostbusters gbr +application/rpki-manifest mft +application/rpki-roa roa +application/rsd+xml rsd +application/rss+xml rss +application/rtf rtf +application/sbml+xml sbml +application/scvp-cv-request scq +application/scvp-cv-response scs +application/scvp-vp-request spq +application/scvp-vp-response spp +application/sdp sdp +application/set-payment-initiation setpay +application/set-registration-initiation setreg +application/shf+xml shf +application/smil+xml smi smil +application/sparql-query rq +application/sparql-results+xml srx +application/srgs gram +application/srgs+xml grxml +application/sru+xml sru +application/ssdl+xml ssdl +application/ssml+xml ssml +application/tei+xml tei teicorpus +application/thraud+xml tfi +application/timestamped-data tsd +application/vnd.3gpp.pic-bw-large plb +application/vnd.3gpp.pic-bw-small psb +application/vnd.3gpp.pic-bw-var pvb +application/vnd.3gpp2.tcap tcap +application/vnd.3m.post-it-notes pwn +application/vnd.accpac.simply.aso aso +application/vnd.accpac.simply.imp imp +application/vnd.acucobol acu +application/vnd.acucorp atc acutc +application/vnd.adobe.air-application-installer-package+zip air +application/vnd.adobe.formscentral.fcdt fcdt +application/vnd.adobe.fxp fxp fxpl +application/vnd.adobe.xdp+xml xdp +application/vnd.adobe.xfdf xfdf +application/vnd.ahead.space ahead +application/vnd.airzip.filesecure.azf azf +application/vnd.airzip.filesecure.azs azs +application/vnd.amazon.ebook azw +application/vnd.americandynamics.acc acc +application/vnd.amiga.ami ami +application/vnd.android.package-archive apk +application/vnd.anser-web-certificate-issue-initiation cii +application/vnd.anser-web-funds-transfer-initiation fti +application/vnd.antix.game-component atx +application/vnd.apple.installer+xml mpkg +application/vnd.apple.mpegurl m3u8 +application/vnd.aristanetworks.swi swi +application/vnd.astraea-software.iota iota +application/vnd.audiograph aep +application/vnd.blueice.multipass mpm +application/vnd.bmi bmi +application/vnd.businessobjects rep +application/vnd.chemdraw+xml cdxml +application/vnd.chipnuts.karaoke-mmd mmd +application/vnd.cinderella cdy +application/vnd.claymore cla +application/vnd.cloanto.rp9 rp9 +application/vnd.clonk.c4group c4g c4d c4f c4p c4u +application/vnd.cluetrust.cartomobile-config c11amc +application/vnd.cluetrust.cartomobile-config-pkg c11amz +application/vnd.commonspace csp +application/vnd.contact.cmsg cdbcmsg +application/vnd.cosmocaller cmc +application/vnd.crick.clicker clkx +application/vnd.crick.clicker.keyboard clkk +application/vnd.crick.clicker.palette clkp +application/vnd.crick.clicker.template clkt +application/vnd.crick.clicker.wordbank clkw +application/vnd.criticaltools.wbs+xml wbs +application/vnd.ctc-posml pml +application/vnd.cups-ppd ppd +application/vnd.curl.car car +application/vnd.curl.pcurl pcurl +application/vnd.dart dart +application/vnd.data-vision.rdz rdz +application/vnd.dece.data uvf uvvf uvd uvvd +application/vnd.dece.ttml+xml uvt uvvt +application/vnd.dece.unspecified uvx uvvx +application/vnd.dece.zip uvz uvvz +application/vnd.denovo.fcselayout-link fe_launch +application/vnd.dna dna +application/vnd.dolby.mlp mlp +application/vnd.dpgraph dpg +application/vnd.dreamfactory dfac +application/vnd.ds-keypoint kpxx +application/vnd.dvb.ait ait +application/vnd.dvb.service svc +application/vnd.dynageo geo +application/vnd.ecowin.chart mag +application/vnd.enliven nml +application/vnd.epson.esf esf +application/vnd.epson.msf msf +application/vnd.epson.quickanime qam +application/vnd.epson.salt slt +application/vnd.epson.ssf ssf +application/vnd.eszigno3+xml es3 et3 +application/vnd.ezpix-album ez2 +application/vnd.ezpix-package ez3 +application/vnd.fdf fdf +application/vnd.fdsn.mseed mseed +application/vnd.fdsn.seed seed dataless +application/vnd.flographit gph +application/vnd.fluxtime.clip ftc +application/vnd.framemaker fm frame maker book +application/vnd.frogans.fnc fnc +application/vnd.frogans.ltf ltf +application/vnd.fsc.weblaunch fsc +application/vnd.fujitsu.oasys oas +application/vnd.fujitsu.oasys2 oa2 +application/vnd.fujitsu.oasys3 oa3 +application/vnd.fujitsu.oasysgp fg5 +application/vnd.fujitsu.oasysprs bh2 +application/vnd.fujixerox.ddd ddd +application/vnd.fujixerox.docuworks xdw +application/vnd.fujixerox.docuworks.binder xbd +application/vnd.fuzzysheet fzs +application/vnd.genomatix.tuxedo txd +application/vnd.geogebra.file ggb +application/vnd.geogebra.tool ggt +application/vnd.geometry-explorer gex gre +application/vnd.geonext gxt +application/vnd.geoplan g2w +application/vnd.geospace g3w +application/vnd.gmx gmx +application/vnd.google-earth.kml+xml kml +application/vnd.google-earth.kmz kmz +application/vnd.grafeq gqf gqs +application/vnd.groove-account gac +application/vnd.groove-help ghf +application/vnd.groove-identity-message gim +application/vnd.groove-injector grv +application/vnd.groove-tool-message gtm +application/vnd.groove-tool-template tpl +application/vnd.groove-vcard vcg +application/vnd.hal+xml hal +application/vnd.handheld-entertainment+xml zmm +application/vnd.hbci hbci +application/vnd.hhe.lesson-player les +application/vnd.hp-hpgl hpgl +application/vnd.hp-hpid hpid +application/vnd.hp-hps hps +application/vnd.hp-jlyt jlt +application/vnd.hp-pcl pcl +application/vnd.hp-pclxl pclxl +application/vnd.hydrostatix.sof-data sfd-hdstx +application/vnd.ibm.minipay mpy +application/vnd.ibm.modcap afp listafp list3820 +application/vnd.ibm.rights-management irm +application/vnd.ibm.secure-container sc +application/vnd.iccprofile icc icm +application/vnd.igloader igl +application/vnd.immervision-ivp ivp +application/vnd.immervision-ivu ivu +application/vnd.insors.igm igm +application/vnd.intercon.formnet xpw xpx +application/vnd.intergeo i2g +application/vnd.intu.qbo qbo +application/vnd.intu.qfx qfx +application/vnd.ipunplugged.rcprofile rcprofile +application/vnd.irepository.package+xml irp +application/vnd.is-xpr xpr +application/vnd.isac.fcs fcs +application/vnd.jam jam +application/vnd.jcp.javame.midlet-rms rms +application/vnd.jisp jisp +application/vnd.joost.joda-archive joda +application/vnd.kahootz ktz ktr +application/vnd.kde.karbon karbon +application/vnd.kde.kchart chrt +application/vnd.kde.kformula kfo +application/vnd.kde.kivio flw +application/vnd.kde.kontour kon +application/vnd.kde.kpresenter kpr kpt +application/vnd.kde.kspread ksp +application/vnd.kde.kword kwd kwt +application/vnd.kenameaapp htke +application/vnd.kidspiration kia +application/vnd.kinar kne knp +application/vnd.koan skp skd skt skm +application/vnd.kodak-descriptor sse +application/vnd.las.las+xml lasxml +application/vnd.llamagraphics.life-balance.desktop lbd +application/vnd.llamagraphics.life-balance.exchange+xml lbe +application/vnd.lotus-1-2-3 123 +application/vnd.lotus-approach apr +application/vnd.lotus-freelance pre +application/vnd.lotus-notes nsf +application/vnd.lotus-organizer org +application/vnd.lotus-screencam scm +application/vnd.lotus-wordpro lwp +application/vnd.macports.portpkg portpkg +application/vnd.mcd mcd +application/vnd.medcalcdata mc1 +application/vnd.mediastation.cdkey cdkey +application/vnd.mfer mwf +application/vnd.mfmp mfm +application/vnd.micrografx.flo flo +application/vnd.micrografx.igx igx +application/vnd.mif mif +application/vnd.mobius.daf daf +application/vnd.mobius.dis dis +application/vnd.mobius.mbk mbk +application/vnd.mobius.mqy mqy +application/vnd.mobius.msl msl +application/vnd.mobius.plc plc +application/vnd.mobius.txf txf +application/vnd.mophun.application mpn +application/vnd.mophun.certificate mpc +application/vnd.mozilla.xul+xml xul +application/vnd.ms-artgalry cil +application/vnd.ms-cab-compressed cab +application/vnd.ms-excel xls xlm xla xlc xlt xlw +application/vnd.ms-excel.addin.macroenabled.12 xlam +application/vnd.ms-excel.sheet.binary.macroenabled.12 xlsb +application/vnd.ms-excel.sheet.macroenabled.12 xlsm +application/vnd.ms-excel.template.macroenabled.12 xltm +application/vnd.ms-fontobject eot +application/vnd.ms-htmlhelp chm +application/vnd.ms-ims ims +application/vnd.ms-lrm lrm +application/vnd.ms-officetheme thmx +application/vnd.ms-pki.seccat cat +application/vnd.ms-pki.stl stl +application/vnd.ms-powerpoint ppt pps pot +application/vnd.ms-powerpoint.addin.macroenabled.12 ppam +application/vnd.ms-powerpoint.presentation.macroenabled.12 pptm +application/vnd.ms-powerpoint.slide.macroenabled.12 sldm +application/vnd.ms-powerpoint.slideshow.macroenabled.12 ppsm +application/vnd.ms-powerpoint.template.macroenabled.12 potm +application/vnd.ms-project mpp mpt +application/vnd.ms-word.document.macroenabled.12 docm +application/vnd.ms-word.template.macroenabled.12 dotm +application/vnd.ms-works wps wks wcm wdb +application/vnd.ms-wpl wpl +application/vnd.ms-xpsdocument xps +application/vnd.mseq mseq +application/vnd.musician mus +application/vnd.muvee.style msty +application/vnd.mynfc taglet +application/vnd.neurolanguage.nlu nlu +application/vnd.nitf ntf nitf +application/vnd.noblenet-directory nnd +application/vnd.noblenet-sealer nns +application/vnd.noblenet-web nnw +application/vnd.nokia.n-gage.data ngdat +application/vnd.nokia.n-gage.symbian.install n-gage +application/vnd.nokia.radio-preset rpst +application/vnd.nokia.radio-presets rpss +application/vnd.novadigm.edm edm +application/vnd.novadigm.edx edx +application/vnd.novadigm.ext ext +application/vnd.oasis.opendocument.chart odc +application/vnd.oasis.opendocument.chart-template otc +application/vnd.oasis.opendocument.database odb +application/vnd.oasis.opendocument.formula odf +application/vnd.oasis.opendocument.formula-template odft +application/vnd.oasis.opendocument.graphics odg +application/vnd.oasis.opendocument.graphics-template otg +application/vnd.oasis.opendocument.image odi +application/vnd.oasis.opendocument.image-template oti +application/vnd.oasis.opendocument.presentation odp +application/vnd.oasis.opendocument.presentation-template otp +application/vnd.oasis.opendocument.spreadsheet ods +application/vnd.oasis.opendocument.spreadsheet-template ots +application/vnd.oasis.opendocument.text odt +application/vnd.oasis.opendocument.text-master odm +application/vnd.oasis.opendocument.text-template ott +application/vnd.oasis.opendocument.text-web oth +application/vnd.olpc-sugar xo +application/vnd.oma.dd2+xml dd2 +application/vnd.openofficeorg.extension oxt +application/vnd.openxmlformats-officedocument.presentationml.presentation pptx +application/vnd.openxmlformats-officedocument.presentationml.slide sldx +application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx +application/vnd.openxmlformats-officedocument.presentationml.template potx +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx +application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx +application/vnd.openxmlformats-officedocument.wordprocessingml.document docx +application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx +application/vnd.osgeo.mapguide.package mgp +application/vnd.osgi.dp dp +application/vnd.osgi.subsystem esa +application/vnd.palm pdb pqa oprc +application/vnd.pawaafile paw +application/vnd.pg.format str +application/vnd.pg.osasli ei6 +application/vnd.picsel efif +application/vnd.pmi.widget wg +application/vnd.pocketlearn plf +application/vnd.powerbuilder6 pbd +application/vnd.previewsystems.box box +application/vnd.proteus.magazine mgz +application/vnd.publishare-delta-tree qps +application/vnd.pvi.ptid1 ptid +application/vnd.quark.quarkxpress qxd qxt qwd qwt qxl qxb +application/vnd.realvnc.bed bed +application/vnd.recordare.musicxml mxl +application/vnd.recordare.musicxml+xml musicxml +application/vnd.rig.cryptonote cryptonote +application/vnd.rim.cod cod +application/vnd.rn-realmedia rm +application/vnd.rn-realmedia-vbr rmvb +application/vnd.route66.link66+xml link66 +application/vnd.sailingtracker.track st +application/vnd.seemail see +application/vnd.sema sema +application/vnd.semd semd +application/vnd.semf semf +application/vnd.shana.informed.formdata ifm +application/vnd.shana.informed.formtemplate itp +application/vnd.shana.informed.interchange iif +application/vnd.shana.informed.package ipk +application/vnd.simtech-mindmapper twd twds +application/vnd.smaf mmf +application/vnd.smart.teacher teacher +application/vnd.solent.sdkm+xml sdkm sdkd +application/vnd.spotfire.dxp dxp +application/vnd.spotfire.sfs sfs +application/vnd.stardivision.calc sdc +application/vnd.stardivision.draw sda +application/vnd.stardivision.impress sdd +application/vnd.stardivision.math smf +application/vnd.stardivision.writer sdw vor +application/vnd.stardivision.writer-global sgl +application/vnd.stepmania.package smzip +application/vnd.stepmania.stepchart sm +application/vnd.sun.xml.calc sxc +application/vnd.sun.xml.calc.template stc +application/vnd.sun.xml.draw sxd +application/vnd.sun.xml.draw.template std +application/vnd.sun.xml.impress sxi +application/vnd.sun.xml.impress.template sti +application/vnd.sun.xml.math sxm +application/vnd.sun.xml.writer sxw +application/vnd.sun.xml.writer.global sxg +application/vnd.sun.xml.writer.template stw +application/vnd.sus-calendar sus susp +application/vnd.svd svd +application/vnd.symbian.install sis sisx +application/vnd.syncml+xml xsm +application/vnd.syncml.dm+wbxml bdm +application/vnd.syncml.dm+xml xdm +application/vnd.tao.intent-module-archive tao +application/vnd.tcpdump.pcap pcap cap dmp +application/vnd.tmobile-livetv tmo +application/vnd.trid.tpt tpt +application/vnd.triscape.mxs mxs +application/vnd.trueapp tra +application/vnd.ufdl ufd ufdl +application/vnd.uiq.theme utz +application/vnd.umajin umj +application/vnd.unity unityweb +application/vnd.uoml+xml uoml +application/vnd.vcx vcx +application/vnd.visio vsd vst vss vsw +application/vnd.visionary vis +application/vnd.vsf vsf +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo wtb +application/vnd.wolfram.player nbp +application/vnd.wordperfect wpd +application/vnd.wqd wqd +application/vnd.wt.stf stf +application/vnd.xara xar +application/vnd.xfdl xfdl +application/vnd.yamaha.hv-dic hvd +application/vnd.yamaha.hv-script hvs +application/vnd.yamaha.hv-voice hvp +application/vnd.yamaha.openscoreformat osf +application/vnd.yamaha.openscoreformat.osfpvg+xml osfpvg +application/vnd.yamaha.smaf-audio saf +application/vnd.yamaha.smaf-phrase spf +application/vnd.yellowriver-custom-menu cmp +application/vnd.zul zir zirz +application/vnd.zzazz.deck+xml zaz +application/voicexml+xml vxml +application/widget wgt +application/winhlp hlp +application/wsdl+xml wsdl +application/wspolicy+xml wspolicy +application/x-7z-compressed 7z +application/x-abiword abw +application/x-ace-compressed ace +application/x-apple-diskimage dmg +application/x-authorware-bin aab x32 u32 vox +application/x-authorware-map aam +application/x-authorware-seg aas +application/x-bcpio bcpio +application/x-bittorrent torrent +application/x-blorb blb blorb +application/x-bzip bz +application/x-bzip2 bz2 boz +application/x-cbr cbr cba cbt cbz cb7 +application/x-cdlink vcd +application/x-cfs-compressed cfs +application/x-chat chat +application/x-chess-pgn pgn +application/x-conference nsc +application/x-cpio cpio +application/x-csh csh +application/x-debian-package deb udeb +application/x-dgc-compressed dgc +application/x-director dir dcr dxr cst cct cxt w3d fgd swa +application/x-doom wad +application/x-dtbncx+xml ncx +application/x-dtbook+xml dtb +application/x-dtbresource+xml res +application/x-dvi dvi +application/x-envoy evy +application/x-eva eva +application/x-font-bdf bdf +application/x-font-ghostscript gsf +application/x-font-linux-psf psf +application/x-font-otf otf +application/x-font-pcf pcf +application/x-font-snf snf +application/x-font-ttf ttf ttc +application/x-font-type1 pfa pfb pfm afm +application/font-woff woff +application/x-freearc arc +application/x-futuresplash spl +application/x-gca-compressed gca +application/x-glulx ulx +application/x-gnumeric gnumeric +application/x-gramps-xml gramps +application/x-gtar gtar +application/x-hdf hdf +application/x-install-instructions install +application/x-iso9660-image iso +application/x-java-jnlp-file jnlp +application/x-latex latex +application/x-lzh-compressed lzh lha +application/x-mie mie +application/x-mobipocket-ebook prc mobi +application/x-ms-application application +application/x-ms-shortcut lnk +application/x-ms-wmd wmd +application/x-ms-wmz wmz +application/x-ms-xbap xbap +application/x-msaccess mdb +application/x-msbinder obd +application/x-mscardfile crd +application/x-msclip clp +application/x-msdownload exe dll com bat msi +application/x-msmediaview mvb m13 m14 +application/x-msmetafile wmf wmz emf emz +application/x-msmoney mny +application/x-mspublisher pub +application/x-msschedule scd +application/x-msterminal trm +application/x-mswrite wri +application/x-netcdf nc cdf +application/x-nzb nzb +application/x-pkcs12 p12 pfx +application/x-pkcs7-certificates p7b spc +application/x-pkcs7-certreqresp p7r +application/x-rar-compressed rar +application/x-research-info-systems ris +application/x-sh sh +application/x-shar shar +application/x-shockwave-flash swf +application/x-silverlight-app xap +application/x-sql sql +application/x-stuffit sit +application/x-stuffitx sitx +application/x-subrip srt +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-t3vm-image t3 +application/x-tads gam +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-tex-tfm tfm +application/x-texinfo texinfo texi +application/x-tgif obj +application/x-ustar ustar +application/x-wais-source src +application/x-x509-ca-cert der crt +application/x-xfig fig +application/x-xliff+xml xlf +application/x-xpinstall xpi +application/x-xz xz +application/x-zmachine z1 z2 z3 z4 z5 z6 z7 z8 +application/xaml+xml xaml +application/xcap-diff+xml xdf +application/xenc+xml xenc +application/xhtml+xml xhtml xht +application/xml xml xsl +application/xml-dtd dtd +application/xop+xml xop +application/xproc+xml xpl +application/xslt+xml xslt +application/xspf+xml xspf +application/xv+xml mxml xhvml xvml xvm +application/yang yang +application/yin+xml yin +application/zip zip +audio/adpcm adp +audio/basic au snd +audio/midi mid midi kar rmi +audio/mp4 mp4a +audio/mpeg mpga mp2 mp2a mp3 m2a m3a +audio/ogg oga ogg spx +audio/s3m s3m +audio/silk sil +audio/vnd.dece.audio uva uvva +audio/vnd.digital-winds eol +audio/vnd.dra dra +audio/vnd.dts dts +audio/vnd.dts.hd dtshd +audio/vnd.lucent.voice lvp +audio/vnd.ms-playready.media.pya pya +audio/vnd.nuera.ecelp4800 ecelp4800 +audio/vnd.nuera.ecelp7470 ecelp7470 +audio/vnd.nuera.ecelp9600 ecelp9600 +audio/vnd.rip rip +audio/webm weba +audio/x-aac aac +audio/x-aiff aif aiff aifc +audio/x-caf caf +audio/x-flac flac +audio/x-matroska mka +audio/x-mpegurl m3u +audio/x-ms-wax wax +audio/x-ms-wma wma +audio/x-pn-realaudio ram ra +audio/x-pn-realaudio-plugin rmp +audio/x-wav wav +audio/xm xm +chemical/x-cdx cdx +chemical/x-cif cif +chemical/x-cmdf cmdf +chemical/x-cml cml +chemical/x-csml csml +chemical/x-xyz xyz +image/bmp bmp +image/cgm cgm +image/g3fax g3 +image/gif gif +image/ief ief +image/jpeg jpeg jpg jpe +image/ktx ktx +image/png png +image/prs.btif btif +image/sgi sgi +image/svg+xml svg svgz +image/tiff tiff tif +image/vnd.adobe.photoshop psd +image/vnd.dece.graphic uvi uvvi uvg uvvg +image/vnd.dvb.subtitle sub +image/vnd.djvu djvu djv +image/vnd.dwg dwg +image/vnd.dxf dxf +image/vnd.fastbidsheet fbs +image/vnd.fpx fpx +image/vnd.fst fst +image/vnd.fujixerox.edmics-mmr mmr +image/vnd.fujixerox.edmics-rlc rlc +image/vnd.ms-modi mdi +image/vnd.ms-photo wdp +image/vnd.net-fpx npx +image/vnd.wap.wbmp wbmp +image/vnd.xiff xif +image/webp webp +image/x-3ds 3ds +image/x-cmu-raster ras +image/x-cmx cmx +image/x-freehand fh fhc fh4 fh5 fh7 +image/x-icon ico +image/x-mrsid-image sid +image/x-pcx pcx +image/x-pict pic pct +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-tga tga +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +message/rfc822 eml mime +model/iges igs iges +model/mesh msh mesh silo +model/vnd.collada+xml dae +model/vnd.dwf dwf +model/vnd.gdl gdl +model/vnd.gtw gtw +model/vnd.mts mts +model/vnd.vtu vtu +model/vrml wrl vrml +model/x3d+binary x3db x3dbz +model/x3d+vrml x3dv x3dvz +model/x3d+xml x3d x3dz +text/cache-manifest appcache +text/calendar ics ifb +text/css css +text/csv csv +text/html html htm +text/n3 n3 +text/plain txt text conf def list log in +text/prs.lines.tag dsc +text/richtext rtx +text/sgml sgml sgm +text/tab-separated-values tsv +text/troff t tr roff man me ms +text/turtle ttl +text/uri-list uri uris urls +text/vcard vcard +text/vnd.curl curl +text/vnd.curl.dcurl dcurl +text/vnd.curl.scurl scurl +text/vnd.curl.mcurl mcurl +text/vnd.dvb.subtitle sub +text/vnd.fly fly +text/vnd.fmi.flexstor flx +text/vnd.graphviz gv +text/vnd.in3d.3dml 3dml +text/vnd.in3d.spot spot +text/vnd.sun.j2me.app-descriptor jad +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/x-asm s asm +text/x-c c cc cxx cpp h hh dic +text/x-fortran f for f77 f90 +text/x-java-source java +text/x-opml opml +text/x-pascal p pas +text/x-nfo nfo +text/x-setext etx +text/x-sfv sfv +text/x-uuencode uu +text/x-vcalendar vcs +text/x-vcard vcf +video/3gpp 3gp +video/3gpp2 3g2 +video/h261 h261 +video/h263 h263 +video/h264 h264 +video/jpeg jpgv +video/jpm jpm jpgm +video/mj2 mj2 mjp2 +video/mp4 mp4 mp4v mpg4 +video/mpeg mpeg mpg mpe m1v m2v +video/ogg ogv +video/quicktime qt mov +video/vnd.dece.hd uvh uvvh +video/vnd.dece.mobile uvm uvvm +video/vnd.dece.pd uvp uvvp +video/vnd.dece.sd uvs uvvs +video/vnd.dece.video uvv uvvv +video/vnd.dvb.file dvb +video/vnd.fvt fvt +video/vnd.mpegurl mxu m4u +video/vnd.ms-playready.media.pyv pyv +video/vnd.uvvu.mp4 uvu uvvu +video/vnd.vivo viv +video/webm webm +video/x-f4v f4v +video/x-fli fli +video/x-flv flv +video/x-m4v m4v +video/x-matroska mkv mk3d mks +video/x-mng mng +video/x-ms-asf asf asx +video/x-ms-vob vob +video/x-ms-wm wm +video/x-ms-wmv wmv +video/x-ms-wmx wmx +video/x-ms-wvx wvx +video/x-msvideo avi +video/x-sgi-movie movie +video/x-smv smv +x-conference/x-cooltalk ice diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h new file mode 100644 index 0000000000..820aa02fbe --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h @@ -0,0 +1,55 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/ByteFlow.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +#if TD_HAVE_OPENSSL +class AesCtrByteFlow : public ByteFlowInplaceBase { + public: + void init(const UInt256 &key, const UInt128 &iv) { + state_.init(key, iv); + } + void init(AesCtrState &&state) { + state_ = std::move(state); + } + AesCtrState move_aes_ctr_state() { + return std::move(state_); + } + void loop() override { + bool was_updated = false; + while (true) { + auto ready = input_->prepare_read(); + if (ready.empty()) { + break; + } + state_.encrypt(ready, MutableSlice(const_cast(ready.data()), ready.size())); + input_->confirm_read(ready.size()); + output_.advance_end(ready.size()); + was_updated = true; + } + if (was_updated) { + on_output_updated(); + } + if (!is_input_active_) { + finish(Status::OK()); // End of input stream. + } + set_need_size(1); + } + + private: + AesCtrState state_; +}; +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp new file mode 100644 index 0000000000..f553661d49 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.cpp @@ -0,0 +1,251 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/BigNum.h" + +char disable_linker_warning_about_empty_file_bignum_cpp TD_UNUSED; + +#if TD_HAVE_OPENSSL + +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +#include +#include + +namespace td { + +class BigNumContext::Impl { + public: + BN_CTX *big_num_context; + + Impl() : big_num_context(BN_CTX_new()) { + LOG_IF(FATAL, big_num_context == nullptr); + } + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl() { + BN_CTX_free(big_num_context); + } +}; + +BigNumContext::BigNumContext() : impl_(make_unique()) { +} + +BigNumContext::BigNumContext(BigNumContext &&other) = default; +BigNumContext &BigNumContext::operator=(BigNumContext &&other) = default; + +BigNumContext::~BigNumContext() = default; + +class BigNum::Impl { + public: + BIGNUM *big_num; + + Impl() : Impl(BN_new()) { + } + explicit Impl(BIGNUM *big_num) : big_num(big_num) { + LOG_IF(FATAL, big_num == nullptr); + } + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl() { + BN_clear_free(big_num); + } +}; + +BigNum::BigNum() : impl_(make_unique()) { +} + +BigNum::BigNum(const BigNum &other) : BigNum() { + *this = other; +} + +BigNum &BigNum::operator=(const BigNum &other) { + CHECK(impl_ != nullptr); + CHECK(other.impl_ != nullptr); + BIGNUM *result = BN_copy(impl_->big_num, other.impl_->big_num); + LOG_IF(FATAL, result == nullptr); + return *this; +} + +BigNum::BigNum(BigNum &&other) = default; + +BigNum &BigNum::operator=(BigNum &&other) = default; + +BigNum::~BigNum() = default; + +BigNum BigNum::from_binary(Slice str) { + return BigNum(make_unique(BN_bin2bn(str.ubegin(), narrow_cast(str.size()), nullptr))); +} + +BigNum BigNum::from_decimal(CSlice str) { + BigNum result; + int err = BN_dec2bn(&result.impl_->big_num, str.c_str()); + LOG_IF(FATAL, err == 0); + return result; +} + +BigNum BigNum::from_raw(void *openssl_big_num) { + return BigNum(make_unique(static_cast(openssl_big_num))); +} + +BigNum::BigNum(unique_ptr &&impl) : impl_(std::move(impl)) { +} + +void BigNum::ensure_const_time() { + BN_set_flags(impl_->big_num, BN_FLG_CONSTTIME); +} + +int BigNum::get_num_bits() const { + return BN_num_bits(impl_->big_num); +} + +int BigNum::get_num_bytes() const { + return BN_num_bytes(impl_->big_num); +} + +void BigNum::set_bit(int num) { + int result = BN_set_bit(impl_->big_num, num); + LOG_IF(FATAL, result != 1); +} + +void BigNum::clear_bit(int num) { + int result = BN_clear_bit(impl_->big_num, num); + LOG_IF(FATAL, result != 1); +} + +bool BigNum::is_bit_set(int num) const { + return BN_is_bit_set(impl_->big_num, num) != 0; +} + +bool BigNum::is_prime(BigNumContext &context) const { + int result = BN_is_prime_ex(impl_->big_num, BN_prime_checks, context.impl_->big_num_context, nullptr); + LOG_IF(FATAL, result == -1); + return result == 1; +} + +void BigNum::operator+=(uint32 value) { + int result = BN_add_word(impl_->big_num, value); + LOG_IF(FATAL, result != 1); +} + +void BigNum::operator-=(uint32 value) { + int result = BN_sub_word(impl_->big_num, value); + LOG_IF(FATAL, result != 1); +} + +void BigNum::operator*=(uint32 value) { + int result = BN_mul_word(impl_->big_num, value); + LOG_IF(FATAL, result != 1); +} + +void BigNum::operator/=(uint32 value) { + BN_ULONG result = BN_div_word(impl_->big_num, value); + LOG_IF(FATAL, result == static_cast(-1)); +} + +uint32 BigNum::operator%(uint32 value) const { + BN_ULONG result = BN_mod_word(impl_->big_num, value); + LOG_IF(FATAL, result == static_cast(-1)); + return narrow_cast(result); +} + +void BigNum::set_value(uint32 new_value) { + if (new_value == 0) { + BN_zero(impl_->big_num); + } else { + int result = BN_set_word(impl_->big_num, new_value); + LOG_IF(FATAL, result != 1); + } +} + +BigNum BigNum::clone() const { + BIGNUM *result = BN_dup(impl_->big_num); + LOG_IF(FATAL, result == nullptr); + return BigNum(make_unique(result)); +} + +string BigNum::to_binary(int exact_size) const { + int num_size = get_num_bytes(); + if (exact_size == -1) { + exact_size = num_size; + } else { + CHECK(exact_size >= num_size); + } + string res(exact_size, '\0'); + BN_bn2bin(impl_->big_num, reinterpret_cast(&res[exact_size - num_size])); + return res; +} + +string BigNum::to_decimal() const { + char *result = BN_bn2dec(impl_->big_num); + CHECK(result != nullptr); + string res(result); + OPENSSL_free(result); + return res; +} + +void BigNum::random(BigNum &r, int bits, int top, int bottom) { + int result = BN_rand(r.impl_->big_num, bits, top, bottom); + LOG_IF(FATAL, result != 1); +} + +void BigNum::add(BigNum &r, const BigNum &a, const BigNum &b) { + int result = BN_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num); + LOG_IF(FATAL, result != 1); +} + +void BigNum::sub(BigNum &r, const BigNum &a, const BigNum &b) { + CHECK(r.impl_->big_num != a.impl_->big_num); + CHECK(r.impl_->big_num != b.impl_->big_num); + int result = BN_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) { + int result = BN_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) { + int result = BN_mod_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num, + context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, + BigNumContext &context) { + auto q = quotient == nullptr ? nullptr : quotient->impl_->big_num; + auto r = remainder == nullptr ? nullptr : remainder->impl_->big_num; + if (q == nullptr && r == nullptr) { + return; + } + + auto result = BN_div(q, r, dividend.impl_->big_num, divisor.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context) { + int result = BN_mod_exp(r.impl_->big_num, a.impl_->big_num, p.impl_->big_num, m.impl_->big_num, + context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) { + int result = BN_gcd(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +int BigNum::compare(const BigNum &a, const BigNum &b) { + return BN_cmp(a.impl_->big_num, b.impl_->big_num); +} + +} // namespace td +#endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h new file mode 100644 index 0000000000..6eecdeab03 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BigNum.h @@ -0,0 +1,108 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#if TD_HAVE_OPENSSL + +#include "td/utils/Slice.h" + +namespace td { + +class BigNumContext { + public: + BigNumContext(); + BigNumContext(const BigNumContext &other) = delete; + BigNumContext &operator=(const BigNumContext &other) = delete; + BigNumContext(BigNumContext &&other); + BigNumContext &operator=(BigNumContext &&other); + ~BigNumContext(); + + private: + class Impl; + unique_ptr impl_; + + friend class BigNum; +}; + +class BigNum { + public: + BigNum(); + BigNum(const BigNum &other); + BigNum &operator=(const BigNum &other); + BigNum(BigNum &&other); + BigNum &operator=(BigNum &&other); + ~BigNum(); + + static BigNum from_binary(Slice str); + + static BigNum from_decimal(CSlice str); + + static BigNum from_raw(void *openssl_big_num); + + void set_value(uint32 new_value); + + void ensure_const_time(); + + int get_num_bits() const; + + int get_num_bytes() const; + + void set_bit(int num); + + void clear_bit(int num); + + bool is_bit_set(int num) const; + + bool is_prime(BigNumContext &context) const; + + BigNum clone() const; + + string to_binary(int exact_size = -1) const; + + string to_decimal() const; + + void operator+=(uint32 value); + + void operator-=(uint32 value); + + void operator*=(uint32 value); + + void operator/=(uint32 value); + + uint32 operator%(uint32 value) const; + + static void random(BigNum &r, int bits, int top, int bottom); + + static void add(BigNum &r, const BigNum &a, const BigNum &b); + + static void sub(BigNum &r, const BigNum &a, const BigNum &b); + + static void mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context); + + static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); + + static void div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, + BigNumContext &context); + + static void mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context); + + static void gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context); + + static int compare(const BigNum &a, const BigNum &b); + + private: + class Impl; + unique_ptr impl_; + + explicit BigNum(unique_ptr &&impl); +}; + +} // namespace td + +#endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h new file mode 100644 index 0000000000..0c8f65408d --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedFd.h @@ -0,0 +1,199 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include + +namespace td { +// just reads from given reader and writes to given writer +template +class BufferedFdBase : public FdT { + public: + BufferedFdBase() = default; + explicit BufferedFdBase(FdT &&fd_); + // TODO: make move constructor and move assignment safer + + Result flush_read(size_t max_read = std::numeric_limits::max()) TD_WARN_UNUSED_RESULT; + Result flush_write() TD_WARN_UNUSED_RESULT; + + bool need_flush_write(size_t at_least = 0) { + CHECK(write_); + write_->sync_with_writer(); + return write_->size() > at_least; + } + size_t ready_for_flush_write() { + CHECK(write_); + write_->sync_with_writer(); + return write_->size(); + } + void set_input_writer(ChainBufferWriter *read) { + read_ = read; + } + void set_output_reader(ChainBufferReader *write) { + write_ = write; + } + + private: + ChainBufferWriter *read_ = nullptr; + ChainBufferReader *write_ = nullptr; +}; + +template +class BufferedFd : public BufferedFdBase { + using Parent = BufferedFdBase; + ChainBufferWriter input_writer_; + ChainBufferReader input_reader_; + ChainBufferWriter output_writer_; + ChainBufferReader output_reader_; + void init(); + void init_ptr(); + + public: + BufferedFd(); + explicit BufferedFd(FdT &&fd_); + BufferedFd(BufferedFd &&); + BufferedFd &operator=(BufferedFd &&); + BufferedFd(const BufferedFd &) = delete; + BufferedFd &operator=(const BufferedFd &) = delete; + ~BufferedFd(); + + void close(); + + Result flush_read(size_t max_read = std::numeric_limits::max()) TD_WARN_UNUSED_RESULT; + Result flush_write() TD_WARN_UNUSED_RESULT; + + // Yep, direct access to buffers. It is IO interface too. + ChainBufferReader &input_buffer(); + ChainBufferWriter &output_buffer(); +}; + +// IMPLEMENTATION + +/*** BufferedFd ***/ +template +BufferedFdBase::BufferedFdBase(FdT &&fd_) : FdT(std::move(fd_)) { +} + +template +Result BufferedFdBase::flush_read(size_t max_read) { + CHECK(read_); + size_t result = 0; + while (::td::can_read(*this) && max_read) { + MutableSlice slice = read_->prepare_append().truncate(max_read); + TRY_RESULT(x, FdT::read(slice)); + slice.truncate(x); + read_->confirm_append(x); + result += x; + max_read -= x; + } + return result; +} + +template +Result BufferedFdBase::flush_write() { + size_t result = 0; + // TODO: sync on demand + write_->sync_with_writer(); + while (!write_->empty() && ::td::can_write(*this)) { + Slice slice = write_->prepare_read(); + TRY_RESULT(x, FdT::write(slice)); + write_->confirm_read(x); + result += x; + } + return result; +} + +/*** BufferedFd ***/ +template +void BufferedFd::init() { + input_reader_ = input_writer_.extract_reader(); + output_reader_ = output_writer_.extract_reader(); + init_ptr(); +} + +template +void BufferedFd::init_ptr() { + this->set_input_writer(&input_writer_); + this->set_output_reader(&output_reader_); +} + +template +BufferedFd::BufferedFd() { + init(); +} + +template +BufferedFd::BufferedFd(FdT &&fd_) : Parent(std::move(fd_)) { + init(); +} + +template +BufferedFd::BufferedFd(BufferedFd &&from) { + *this = std::move(from); +} + +template +BufferedFd &BufferedFd::operator=(BufferedFd &&from) { + FdT::operator=(std::move(static_cast(from))); + input_reader_ = std::move(from.input_reader_); + input_writer_ = std::move(from.input_writer_); + output_reader_ = std::move(from.output_reader_); + output_writer_ = std::move(from.output_writer_); + init_ptr(); + return *this; +} + +template +BufferedFd::~BufferedFd() { + close(); +} + +template +void BufferedFd::close() { + FdT::close(); + // TODO: clear buffers +} + +template +Result BufferedFd::flush_read(size_t max_read) { + TRY_RESULT(result, Parent::flush_read(max_read)); + if (result) { + // TODO: faster sync is possible if you owns writer. + input_reader_.sync_with_writer(); + LOG(DEBUG) << "flush_read: +" << format::as_size(result) << tag("total", format::as_size(input_reader_.size())); + } + return result; +} + +template +Result BufferedFd::flush_write() { + TRY_RESULT(result, Parent::flush_write()); + if (result) { + LOG(DEBUG) << "flush_write: +" << format::as_size(result) << tag("left", format::as_size(output_reader_.size())); + } + return result; +} + +// Yep, direct access to buffers. It is IO interface too. +template +ChainBufferReader &BufferedFd::input_buffer() { + return input_reader_; +} + +template +ChainBufferWriter &BufferedFd::output_buffer() { + return output_writer_; +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h new file mode 100644 index 0000000000..9006d78132 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/BufferedReader.h @@ -0,0 +1,61 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include + +namespace td { +class BufferedReader { + public: + explciit BufferedReader(FileFd &file, size_t buff_size = 8152) + : file_(file), buff_(buff_size), begin_pos_(0), end_pos_(0) { + } + + Result read(MutableSlice slice) TD_WARN_UNUSED_RESULT; + + private: + FileFd &file_; + vector buff_; + size_t begin_pos_; + size_t end_pos_; +}; + +inline Result BufferedReader::read(MutableSlice slice) { + size_t available = end_pos_ - begin_pos_; + if (available >= slice.size()) { + // have enough data in buffer + std::memcpy(slice.begin(), &buff_[begin_pos_], slice.size()); + begin_pos_ += slice.size(); + return slice.size(); + } + + if (available) { + std::memcpy(slice.begin(), &buff_[begin_pos_], available); + begin_pos_ += available; + slice.remove_prefix(available); + } + + if (slice.size() > buff_.size() / 2) { + TRY_RESULT(result, file_.read(slice)); + return result + available; + } + + TRY_RESULT(result, file_.read({&buff_[0], buff_.size()})); + begin_pos_ = 0; + end_pos_ = result; + + size_t left = min(end_pos_, slice.size()); + std::memcpy(slice.begin(), &buff_[begin_pos_], left); + begin_pos_ += left; + return left + available; +} +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h new file mode 100644 index 0000000000..fb0c4489eb --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ByteFlow.h @@ -0,0 +1,288 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +namespace td { + +class ByteFlowInterface { + public: + virtual void close_input(Status status) = 0; + virtual void wakeup() = 0; + virtual void set_parent(ByteFlowInterface &other) = 0; + virtual void set_input(ChainBufferReader *input) = 0; + virtual size_t get_need_size() = 0; + ByteFlowInterface() = default; + ByteFlowInterface(const ByteFlowInterface &) = delete; + ByteFlowInterface &operator=(const ByteFlowInterface &) = delete; + ByteFlowInterface(ByteFlowInterface &&) = default; + ByteFlowInterface &operator=(ByteFlowInterface &&) = default; + virtual ~ByteFlowInterface() = default; +}; + +class ByteFlowBaseCommon : public ByteFlowInterface { + public: + ByteFlowBaseCommon() = default; + + void close_input(Status status) final { + if (status.is_error()) { + finish(std::move(status)); + } else { + is_input_active_ = false; + wakeup(); + } + } + + void wakeup() final { + if (stop_flag_) { + return; + } + input_->sync_with_writer(); + if (waiting_flag_) { + if (!is_input_active_) { + finish(Status::OK()); + } + return; + } + if (is_input_active_) { + if (need_size_ != 0 && input_->size() < need_size_) { + return; + } + } + need_size_ = 0; + loop(); + } + + size_t get_need_size() final { + return need_size_; + } + + virtual void loop() = 0; + + protected: + bool waiting_flag_ = false; + ChainBufferReader *input_; + bool is_input_active_ = true; + size_t need_size_ = 0; + void finish(Status status) { + stop_flag_ = true; + need_size_ = 0; + if (parent_) { + parent_->close_input(std::move(status)); + parent_ = nullptr; + } + } + + void set_need_size(size_t need_size) { + need_size_ = need_size; + } + + void on_output_updated() { + if (parent_) { + parent_->wakeup(); + } + } + void consume_input() { + waiting_flag_ = true; + if (!is_input_active_) { + finish(Status::OK()); + } + } + + private: + ByteFlowInterface *parent_ = nullptr; + bool stop_flag_ = false; + friend class ByteFlowBase; + friend class ByteFlowInplaceBase; +}; + +class ByteFlowBase : public ByteFlowBaseCommon { + public: + ByteFlowBase() = default; + + void set_input(ChainBufferReader *input) final { + input_ = input; + } + void set_parent(ByteFlowInterface &other) final { + parent_ = &other; + parent_->set_input(&output_reader_); + } + void loop() override = 0; + + // ChainBufferWriter &get_output() { + // return output_; + //} + + protected: + ChainBufferWriter output_; + ChainBufferReader output_reader_ = output_.extract_reader(); +}; + +class ByteFlowInplaceBase : public ByteFlowBaseCommon { + public: + ByteFlowInplaceBase() = default; + + void set_input(ChainBufferReader *input) final { + input_ = input; + output_ = ChainBufferReader(input_->begin().clone(), input_->begin().clone(), false); + } + void set_parent(ByteFlowInterface &other) final { + parent_ = &other; + parent_->set_input(&output_); + } + void loop() override = 0; + + ChainBufferReader &get_output() { + return output_; + } + + protected: + ChainBufferReader output_; +}; + +inline ByteFlowInterface &operator>>(ByteFlowInterface &from, ByteFlowInterface &to) { + from.set_parent(to); + return to; +} + +class ByteFlowSource : public ByteFlowInterface { + public: + ByteFlowSource() = default; + explicit ByteFlowSource(ChainBufferReader *buffer) : buffer_(buffer) { + } + ByteFlowSource(ByteFlowSource &&other) : buffer_(other.buffer_), parent_(other.parent_) { + other.buffer_ = nullptr; + other.parent_ = nullptr; + } + ByteFlowSource &operator=(ByteFlowSource &&other) { + buffer_ = other.buffer_; + parent_ = other.parent_; + other.buffer_ = nullptr; + other.parent_ = nullptr; + return *this; + } + ByteFlowSource(const ByteFlowSource &) = delete; + ByteFlowSource &operator=(const ByteFlowSource &) = delete; + ~ByteFlowSource() override = default; + + void set_input(ChainBufferReader *) final { + UNREACHABLE(); + } + void set_parent(ByteFlowInterface &parent) final { + CHECK(parent_ == nullptr); + parent_ = &parent; + parent_->set_input(buffer_); + } + void close_input(Status status) final { + CHECK(parent_); + parent_->close_input(std::move(status)); + parent_ = nullptr; + } + void wakeup() final { + CHECK(parent_); + parent_->wakeup(); + } + size_t get_need_size() final { + if (parent_ == nullptr) { + return 0; + } + return parent_->get_need_size(); + } + + private: + ChainBufferReader *buffer_ = nullptr; + ByteFlowInterface *parent_ = nullptr; +}; + +class ByteFlowSink : public ByteFlowInterface { + public: + void set_input(ChainBufferReader *input) final { + CHECK(buffer_ == nullptr); + buffer_ = input; + } + void set_parent(ByteFlowInterface &parent) final { + UNREACHABLE(); + } + void close_input(Status status) final { + CHECK(active_); + active_ = false; + status_ = std::move(status); + buffer_->sync_with_writer(); + } + void wakeup() final { + buffer_->sync_with_writer(); + } + size_t get_need_size() final { + UNREACHABLE(); + return 0; + } + bool is_ready() { + return !active_; + } + Status &status() { + return status_; + } + ChainBufferReader *result() { + CHECK(is_ready() && status().is_ok()); + return buffer_; + } + ChainBufferReader *get_output() { + return buffer_; + } + + private: + bool active_ = true; + Status status_; + ChainBufferReader *buffer_ = nullptr; +}; + +class ByteFlowMoveSink : public ByteFlowInterface { + public: + void set_input(ChainBufferReader *input) final { + CHECK(!input_); + input_ = input; + } + void set_parent(ByteFlowInterface &parent) final { + UNREACHABLE(); + } + void close_input(Status status) final { + CHECK(active_); + active_ = false; + status_ = std::move(status); + wakeup(); + } + void wakeup() final { + input_->sync_with_writer(); + output_->append(*input_); + } + size_t get_need_size() final { + UNREACHABLE(); + return 0; + } + void set_output(ChainBufferWriter *output) { + CHECK(!output_); + output_ = output; + } + + bool is_ready() { + return !active_; + } + Status &status() { + return status_; + } + + private: + bool active_ = true; + Status status_; + ChainBufferReader *input_ = nullptr; + ChainBufferWriter *output_ = nullptr; +}; +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h new file mode 100644 index 0000000000..9342f45a8b --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ChangesProcessor.h @@ -0,0 +1,61 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include + +namespace td { + +// Process changes after they are finished in order of addition +template +class ChangesProcessor { + public: + using Id = uint64; + + void clear() { + offset_ += data_array_.size(); + ready_i_ = 0; + data_array_.clear(); + } + + template + Id add(FromDataT &&data) { + auto res = offset_ + data_array_.size(); + data_array_.emplace_back(std::forward(data), false); + return static_cast(res); + } + + template + void finish(Id token, F &&func) { + size_t pos = static_cast(token) - offset_; + if (pos >= data_array_.size()) { + return; + } + data_array_[pos].second = true; + while (ready_i_ < data_array_.size() && data_array_[ready_i_].second == true) { + func(std::move(data_array_[ready_i_].first)); + ready_i_++; + } + try_compactify(); + } + + private: + size_t offset_ = 1; + size_t ready_i_ = 0; + std::vector> data_array_; + void try_compactify() { + if (ready_i_ > 5 && ready_i_ * 2 > data_array_.size()) { + data_array_.erase(data_array_.begin(), data_array_.begin() + ready_i_); + offset_ += ready_i_; + ready_i_ = 0; + } + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h new file mode 100644 index 0000000000..718f930b8a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Closure.h @@ -0,0 +1,169 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/logging.h" + +#include +#include +#include +#include + +// +// Essentially we have: +// (ActorT::func, arg1, arg2, ..., argn) +// We want to call: +// actor->func(arg1, arg2, ..., argn) +// And in some cases we would like to delay this call. +// +// First attempt would be +// [a1=arg1, a2=arg2, ..., an=argn](ActorT *actor) { +// actor->func(a1, a2, ..., an) +// } +// +// But there are some difficulties with elimitation on unnecessary copies. +// We want to use move constructor when it is possible +// +// We may pass +// Tmp. Temporary / rvalue reference +// Var. Variable / reference +// CnstRef. const reference +// +// +// Function may expect +// Val. Value +// CnstRef. const reference +// Ref. rvalue reverence / reference +// +// TODO: +// Immediate call / Delayed call +// Tmp->Val move / move->move +// Tmp->CnstRef + / move->+ +// Tmp->Ref + / move->+ +// Var->Val copy / copy->move +// Var->CnstRef + / copy-> +// Var->Ref + / copy->+ // khm. It will complile, but won't work +// +// So I will use common idiom: forward references +// If delay is needed, just std::forward data to temporary storage, and std::move them when call is executed. +// +// +// create_immediate_closure(&Actor::func, arg1, arg2, ..., argn).run(actor) +// to_delayed_closure(std::move(immediate)).run(actor) + +namespace td { +template +class DelayedClosure; + +template +class ImmediateClosure { + public: + using Delayed = DelayedClosure; + friend Delayed; + using ActorType = ActorT; + + void run(ActorT *actor) { + mem_call_tuple(actor, func, std::move(args)); + } + + // no &&. just save references as references. + explicit ImmediateClosure(FunctionT func, ArgsT... args) : func(func), args(std::forward(args)...) { + } + + private: + FunctionT func; + std::tuple args; +}; + +template +ImmediateClosure create_immediate_closure( + ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { + return ImmediateClosure(func, + std::forward(args)...); +} + +template +class DelayedClosure { + public: + using ActorType = ActorT; + using Delayed = DelayedClosure; + + void run(ActorT *actor) { + mem_call_tuple(actor, func, std::move(args)); + } + + DelayedClosure clone() const { + return do_clone(*this); + } + + explicit DelayedClosure(ImmediateClosure &&other) + : func(std::move(other.func)), args(std::move(other.args)) { + } + + explicit DelayedClosure(FunctionT func, ArgsT... args) : func(func), args(std::forward(args)...) { + } + + template + void for_each(const F &f) { + tuple_for_each(args, f); + } + + private: + using ArgsStorageT = std::tuple::type...>; + + FunctionT func; + ArgsStorageT args; + + template + explicit DelayedClosure(const DelayedClosure &other, + std::enable_if_t::value...>::value, int> = 0) + : func(other.func), args(other.args) { + } + + template + explicit DelayedClosure( + const DelayedClosure &other, + std::enable_if_t::value...>::value, int> = 0) { + LOG(FATAL) << "Deleted constructor"; + std::abort(); + } + + template + std::enable_if_t::value...>::value, + DelayedClosure> + do_clone(const DelayedClosure &value) const { + LOG(FATAL) << "Trying to clone DelayedClosure that contains noncopyable elements"; + std::abort(); + } + + template + std::enable_if_t::value...>::value, + DelayedClosure> + do_clone(const DelayedClosure &value) const { + return DelayedClosure(value); + } +}; + +template +typename ImmediateClosure::Delayed to_delayed_closure(ImmediateClosure &&other) { + return typename ImmediateClosure::Delayed(std::move(other)); +} + +template +DelayedClosure to_delayed_closure(DelayedClosure &&other) { + return std::move(other); +} + +template +auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { + return DelayedClosure(func, + std::forward(args)...); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h new file mode 100644 index 0000000000..57b4bb4d16 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Container.h @@ -0,0 +1,149 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include + +namespace td { + +// 1. Allocates all objects in vector. (but vector never shrinks) +// 2. Id is safe way to reach this object. +// 3. All ids are unique. +// 4. All ids are non-zero. +template +class Container { + public: + using Id = uint64; + DataT *get(Id id) { + int32 slot_id = decode_id(id); + if (slot_id == -1) { + return nullptr; + } + return &slots_[slot_id].data; + } + + void erase(Id id) { + int32 slot_id = decode_id(id); + if (slot_id == -1) { + return; + } + release(slot_id); + } + + DataT extract(Id id) { + int32 slot_id = decode_id(id); + CHECK(slot_id != -1); + auto res = std::move(slots_[slot_id].data); + release(slot_id); + return res; + } + + Id create(DataT &&data = DataT(), uint8 type = 0) { + int32 id = store(std::move(data), type); + return encode_id(id); + } + + Id reset_id(Id id) { + int32 slot_id = decode_id(id); + CHECK(slot_id != -1); + inc_generation(slot_id); + return encode_id(slot_id); + } + + static uint8 type_from_id(Id id) { + return static_cast(id); + } + + vector ids() { + vector is_bad(slots_.size(), false); + for (auto id : empty_slots_) { + is_bad[id] = true; + } + vector res; + for (size_t i = 0, n = slots_.size(); i < n; i++) { + if (!is_bad[i]) { + res.push_back(encode_id(static_cast(i))); + } + } + return res; + } + template + void for_each(const F &f) { + auto ids = this->ids(); + for (auto id : ids) { + f(id, *get(id)); + } + } + size_t size() const { + CHECK(empty_slots_.size() <= slots_.size()); + return slots_.size() - empty_slots_.size(); + } + bool empty() const { + return size() == 0; + } + void clear() { + *this = Container(); + } + + private: + static constexpr uint32 GENERATION_STEP = 1 << 8; + static constexpr uint32 TYPE_MASK = (1 << 8) - 1; + struct Slot { + uint32 generation; + DataT data; + }; + vector slots_; + vector empty_slots_; + + Id encode_id(int32 id) const { + return (static_cast(id) << 32) | slots_[id].generation; + } + + int32 decode_id(Id id) const { + int32 slot_id = static_cast(id >> 32); + uint32 generation = static_cast(id); + if (slot_id < 0 || slot_id >= static_cast(slots_.size())) { + return -1; + } + if (generation != slots_[slot_id].generation) { + return -1; + } + return slot_id; + } + + int32 store(DataT &&data, uint8 type) { + int32 pos; + if (!empty_slots_.empty()) { + pos = empty_slots_.back(); + empty_slots_.pop_back(); + slots_[pos].data = std::move(data); + slots_[pos].generation ^= (slots_[pos].generation & TYPE_MASK) ^ type; + } else { + CHECK(slots_.size() <= static_cast(std::numeric_limits::max())); + pos = static_cast(slots_.size()); + slots_.push_back(Slot{GENERATION_STEP + type, std::move(data)}); + } + return pos; + } + + void release(int32 id) { + inc_generation(id); + slots_[id].data = DataT(); + if (slots_[id].generation & ~TYPE_MASK) { // generation overflow. Can't use this id anymore + empty_slots_.push_back(id); + } + } + + void inc_generation(int32 id) { + slots_[id].generation += GENERATION_STEP; + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h new file mode 100644 index 0000000000..ca7c0493ff --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Enumerator.h @@ -0,0 +1,45 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +#include +#include + +namespace td { + +template +class Enumerator { + public: + using Key = int32; + + Key add(ValueT v) { + int32 next_id = narrow_cast(arr_.size() + 1); + bool was_inserted; + decltype(map_.begin()) it; + std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id); + if (was_inserted) { + arr_.push_back(&it->first); + } + return it->second; + } + + const ValueT &get(Key key) const { + auto pos = static_cast(key - 1); + CHECK(pos < arr_.size()); + return *arr_[pos]; + } + + private: + std::map map_; + std::vector arr_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp new file mode 100644 index 0000000000..e3c84f1713 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.cpp @@ -0,0 +1,92 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/FileLog.h" + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/port/path.h" +#include "td/utils/Slice.h" + +#include + +namespace td { + +bool FileLog::init(string path, int64 rotate_threshold) { + if (path == path_) { + set_rotate_threshold(rotate_threshold); + return true; + } + + auto r_fd = FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append); + if (r_fd.is_error()) { + LOG(ERROR) << "Can't open log: " << r_fd.error(); + return false; + } + + fd_.close(); + fd_ = r_fd.move_as_ok(); + Fd::duplicate(fd_.get_fd(), Fd::Stderr()).ignore(); + + path_ = std::move(path); + size_ = fd_.get_size(); + rotate_threshold_ = rotate_threshold; + return true; +} + +void FileLog::set_rotate_threshold(int64 rotate_threshold) { + rotate_threshold_ = rotate_threshold; +} + +void FileLog::append(CSlice cslice, int log_level) { + Slice slice = cslice; + while (!slice.empty()) { + auto r_size = fd_.write(slice); + if (r_size.is_error()) { + process_fatal_error(r_size.error().message()); + } + auto written = r_size.ok(); + size_ += static_cast(written); + slice.remove_prefix(written); + } + if (log_level == VERBOSITY_NAME(FATAL)) { + process_fatal_error(cslice); + } + + if (size_ > rotate_threshold_) { + auto status = rename(path_, path_ + ".old"); + if (status.is_error()) { + process_fatal_error(status.message()); + } + do_rotate(); + } +} + +void FileLog::rotate() { + if (path_.empty()) { + return; + } + do_rotate(); +} + +void FileLog::do_rotate() { + auto current_verbosity_level = GET_VERBOSITY_LEVEL(); + SET_VERBOSITY_LEVEL(std::numeric_limits::min()); // to ensure that nothing will be printed to the closed log + CHECK(!path_.empty()); + fd_.close(); + auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Truncate | FileFd::Write); + if (r_fd.is_error()) { + process_fatal_error(r_fd.error().message()); + } + fd_ = r_fd.move_as_ok(); + Fd::duplicate(fd_.get_fd(), Fd::Stderr()).ignore(); + size_ = 0; + SET_VERBOSITY_LEVEL(current_verbosity_level); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h new file mode 100644 index 0000000000..12e9d1479a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FileLog.h @@ -0,0 +1,37 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" + +namespace td { + +class FileLog : public LogInterface { + static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20); + + public: + bool init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD); + + void set_rotate_threshold(int64 rotate_threshold); + + void append(CSlice cslice, int log_level) override; + + void rotate() override; + + private: + FileFd fd_; + string path_; + int64 size_; + int64 rotate_threshold_; + + void do_rotate(); +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h new file mode 100644 index 0000000000..9f047881aa --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlFast.h @@ -0,0 +1,62 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/TimedStat.h" + +namespace td { + +class FloodControlFast { + public: + uint32 add_event(int32 now) { + for (auto &limit : limits_) { + limit.stat_.add_event(CounterStat::Event(), now); + if (limit.stat_.get_stat(now).count_ > limit.count_) { + wakeup_at_ = max(wakeup_at_, now + limit.duration_ * 2); + } + } + return wakeup_at_; + } + uint32 get_wakeup_at() { + return wakeup_at_; + } + + void add_limit(uint32 duration, int32 count) { + limits_.push_back({TimedStat(duration, 0), duration, count}); + } + + void clear_events() { + for (auto &limit : limits_) { + limit.stat_.clear_events(); + } + wakeup_at_ = 0; + } + + private: + class CounterStat { + public: + struct Event {}; + int32 count_ = 0; + void on_event(Event e) { + count_++; + } + void clear() { + count_ = 0; + } + }; + + uint32 wakeup_at_ = 0; + struct Limit { + TimedStat stat_; + uint32 duration_; + int32 count_; + }; + std::vector limits_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h new file mode 100644 index 0000000000..521fbbedc0 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/FloodControlStrict.h @@ -0,0 +1,97 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include + +namespace td { + +// More strict implementaions of flood control than FloodControlFast. +// Should be just fine for small counters. +class FloodControlStrict { + public: + int32 add_event(int32 now) { + events_.push_back(Event{now}); + if (without_update_ > 0) { + without_update_--; + } else { + update(now); + } + return wakeup_at_; + } + + // no more than count in each duration. + void add_limit(int32 duration, int32 count) { + limits_.push_back(Limit{duration, count, 0}); + } + + int32 get_wakeup_at() { + return wakeup_at_; + } + + void clear_events() { + events_.clear(); + for (auto &limit : limits_) { + limit.pos_ = 0; + } + without_update_ = 0; + wakeup_at_ = 0; + } + + int32 update(int32 now) { + size_t min_pos = events_.size(); + + without_update_ = std::numeric_limits::max(); + for (auto &limit : limits_) { + if (limit.pos_ + limit.count_ < events_.size()) { + limit.pos_ = events_.size() - limit.count_; + } + + // binary-search? :D + while (limit.pos_ < events_.size() && events_[limit.pos_].timestamp_ + limit.duration_ < now) { + limit.pos_++; + } + + if (limit.count_ + limit.pos_ <= events_.size()) { + CHECK(limit.count_ + limit.pos_ == events_.size()); + wakeup_at_ = max(wakeup_at_, events_[limit.pos_].timestamp_ + limit.duration_); + without_update_ = 0; + } else { + without_update_ = min(without_update_, limit.count_ + limit.pos_ - events_.size()); + } + + min_pos = min(min_pos, limit.pos_); + } + + if (min_pos * 2 > events_.size()) { + for (auto &limit : limits_) { + limit.pos_ -= min_pos; + } + events_.erase(events_.begin(), events_.begin() + min_pos); + } + return wakeup_at_; + } + + private: + int32 wakeup_at_ = 0; + struct Event { + int32 timestamp_; + }; + struct Limit { + int32 duration_; + int32 count_; + size_t pos_; + }; + size_t without_update_ = 0; + std::vector events_; + std::vector limits_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp new file mode 100644 index 0000000000..976286b923 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.cpp @@ -0,0 +1,20 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/GitInfo.h" + +#include "auto/git_info.h" + +namespace td { + +CSlice GitInfo::commit() { + return GIT_COMMIT; +} +bool GitInfo::is_dirty() { + return GIT_DIRTY; +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h new file mode 100644 index 0000000000..a3ba32602f --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/GitInfo.h @@ -0,0 +1,19 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Slice.h" + +namespace td { + +class GitInfo { + public: + static CSlice commit(); + static bool is_dirty(); +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp new file mode 100644 index 0000000000..d4e60d6e29 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.cpp @@ -0,0 +1,191 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Gzip.h" + +char disable_linker_warning_about_empty_file_gzip_cpp TD_UNUSED; + +#if TD_HAVE_ZLIB +#include "td/utils/logging.h" + +#include +#include + +#include + +namespace td { + +class Gzip::Impl { + public: + z_stream stream_; + + // z_stream is not copyable nor movable + Impl() = default; + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl() = default; +}; + +Status Gzip::init_encode() { + CHECK(mode_ == Empty); + init_common(); + mode_ = Encode; + int ret = deflateInit2(&impl_->stream_, 6, Z_DEFLATED, 15, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + return Status::Error("zlib deflate init failed"); + } + return Status::OK(); +} + +Status Gzip::init_decode() { + CHECK(mode_ == Empty); + init_common(); + mode_ = Decode; + int ret = inflateInit2(&impl_->stream_, MAX_WBITS + 32); + if (ret != Z_OK) { + return Status::Error("zlib inflate init failed"); + } + return Status::OK(); +} + +void Gzip::set_input(Slice input) { + CHECK(input_size_ == 0); + CHECK(!close_input_flag_); + CHECK(input.size() <= std::numeric_limits::max()); + CHECK(impl_->stream_.avail_in == 0); + input_size_ = input.size(); + impl_->stream_.avail_in = static_cast(input.size()); + impl_->stream_.next_in = reinterpret_cast(const_cast(input.data())); +} + +void Gzip::set_output(MutableSlice output) { + CHECK(output_size_ == 0); + CHECK(output.size() <= std::numeric_limits::max()); + CHECK(impl_->stream_.avail_out == 0); + output_size_ = output.size(); + impl_->stream_.avail_out = static_cast(output.size()); + impl_->stream_.next_out = reinterpret_cast(output.data()); +} + +Result Gzip::run() { + while (true) { + int ret; + if (mode_ == Decode) { + ret = inflate(&impl_->stream_, Z_NO_FLUSH); + } else { + ret = deflate(&impl_->stream_, close_input_flag_ ? Z_FINISH : Z_NO_FLUSH); + } + + if (ret == Z_OK) { + return Running; + } + if (ret == Z_STREAM_END) { + // TODO(now): fail if input is not empty; + clear(); + return Done; + } + clear(); + return Status::Error(PSLICE() << "zlib error " << ret); + } +} + +size_t Gzip::left_input() const { + return impl_->stream_.avail_in; +} +size_t Gzip::left_output() const { + return impl_->stream_.avail_out; +} + +void Gzip::init_common() { + std::memset(&impl_->stream_, 0, sizeof(impl_->stream_)); + impl_->stream_.zalloc = Z_NULL; + impl_->stream_.zfree = Z_NULL; + impl_->stream_.opaque = Z_NULL; + impl_->stream_.avail_in = 0; + impl_->stream_.next_in = nullptr; + impl_->stream_.avail_out = 0; + impl_->stream_.next_out = nullptr; + + input_size_ = 0; + output_size_ = 0; + + close_input_flag_ = false; +} + +void Gzip::clear() { + if (mode_ == Decode) { + inflateEnd(&impl_->stream_); + } else if (mode_ == Encode) { + deflateEnd(&impl_->stream_); + } + mode_ = Empty; +} + +Gzip::Gzip() : impl_(make_unique()) { +} + +Gzip::Gzip(Gzip &&other) = default; + +Gzip &Gzip::operator=(Gzip &&other) = default; + +Gzip::~Gzip() { + clear(); +} + +BufferSlice gzdecode(Slice s) { + Gzip gzip; + gzip.init_decode().ensure(); + auto message = ChainBufferWriter::create_empty(); + gzip.set_input(s); + gzip.close_input(); + double k = 2; + gzip.set_output(message.prepare_append(static_cast(static_cast(s.size()) * k))); + while (true) { + auto r_state = gzip.run(); + if (r_state.is_error()) { + return BufferSlice(); + } + auto state = r_state.ok(); + if (state == Gzip::Done) { + message.confirm_append(gzip.flush_output()); + break; + } + if (gzip.need_input()) { + return BufferSlice(); + } + if (gzip.need_output()) { + message.confirm_append(gzip.flush_output()); + k *= 1.5; + gzip.set_output(message.prepare_append(static_cast(static_cast(gzip.left_input()) * k))); + } + } + return message.extract_reader().move_as_buffer_slice(); +} + +BufferSlice gzencode(Slice s, double k) { + Gzip gzip; + gzip.init_encode().ensure(); + gzip.set_input(s); + gzip.close_input(); + size_t max_size = static_cast(static_cast(s.size()) * k); + BufferWriter message{max_size}; + gzip.set_output(message.prepare_append()); + auto r_state = gzip.run(); + if (r_state.is_error()) { + return BufferSlice(); + } + auto state = r_state.ok(); + if (state != Gzip::Done) { + return BufferSlice(); + } + message.confirm_append(gzip.flush_output()); + return message.as_buffer_slice(); +} + +} // namespace td +#endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h new file mode 100644 index 0000000000..dd5fba5bf5 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Gzip.h @@ -0,0 +1,104 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#if TD_HAVE_ZLIB +#include "td/utils/buffer.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +class Gzip { + public: + Gzip(); + Gzip(const Gzip &) = delete; + Gzip &operator=(const Gzip &) = delete; + Gzip(Gzip &&other); + Gzip &operator=(Gzip &&other); + ~Gzip(); + + enum Mode { Empty, Encode, Decode }; + Status init(Mode mode) TD_WARN_UNUSED_RESULT { + if (mode == Encode) { + return init_encode(); + } else if (mode == Decode) { + return init_decode(); + } + clear(); + return Status::OK(); + } + + Status init_encode() TD_WARN_UNUSED_RESULT; + + Status init_decode() TD_WARN_UNUSED_RESULT; + + void set_input(Slice input); + + void set_output(MutableSlice output); + + void close_input() { + close_input_flag_ = true; + } + + bool need_input() const { + return left_input() == 0; + } + + bool need_output() const { + return left_output() == 0; + } + + size_t left_input() const; + + size_t left_output() const; + + size_t used_input() const { + return input_size_ - left_input(); + } + + size_t used_output() const { + return output_size_ - left_output(); + } + + size_t flush_input() { + auto res = used_input(); + input_size_ = left_input(); + return res; + } + + size_t flush_output() { + auto res = used_output(); + output_size_ = left_output(); + return res; + } + + enum State { Running, Done }; + Result run() TD_WARN_UNUSED_RESULT; + + private: + class Impl; + unique_ptr impl_; + + size_t input_size_ = 0; + size_t output_size_ = 0; + bool close_input_flag_ = false; + Mode mode_ = Empty; + + void init_common(); + void clear(); +}; + +BufferSlice gzdecode(Slice s); + +BufferSlice gzencode(Slice s, double k = 0.9); + +} // namespace td + +#endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp new file mode 100644 index 0000000000..d225ef800e --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp @@ -0,0 +1,70 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/GzipByteFlow.h" + +char disable_linker_warning_about_empty_file_gzipbyteflow_cpp TD_UNUSED; + +#if TD_HAVE_ZLIB +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +namespace td { + +void GzipByteFlow::loop() { + while (true) { + if (gzip_.need_input()) { + auto slice = input_->prepare_read(); + if (slice.empty()) { + if (!is_input_active_) { + gzip_.close_input(); + } else { + break; + } + } else { + gzip_.set_input(input_->prepare_read()); + } + } + if (gzip_.need_output()) { + auto slice = output_.prepare_append(); + CHECK(!slice.empty()); + gzip_.set_output(slice); + } + auto r_state = gzip_.run(); + auto output_size = gzip_.flush_output(); + if (output_size) { + uncommited_size_ += output_size; + total_output_size_ += output_size; + if (total_output_size_ > max_output_size_) { + return finish(Status::Error("Max output size limit exceeded")); + } + output_.confirm_append(output_size); + } + + auto input_size = gzip_.flush_input(); + if (input_size) { + input_->confirm_read(input_size); + } + if (r_state.is_error()) { + return finish(r_state.move_as_error()); + } + auto state = r_state.ok(); + if (state == Gzip::Done) { + on_output_updated(); + return consume_input(); + } + } + if (uncommited_size_ >= MIN_UPDATE_SIZE) { + uncommited_size_ = 0; + on_output_updated(); + } +} + +constexpr size_t GzipByteFlow::MIN_UPDATE_SIZE; + +} // namespace td + +#endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h new file mode 100644 index 0000000000..c7e07abd0a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/GzipByteFlow.h @@ -0,0 +1,48 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/ByteFlow.h" +#include "td/utils/Gzip.h" + +#include + +namespace td { + +#if TD_HAVE_ZLIB +class GzipByteFlow final : public ByteFlowBase { + public: + GzipByteFlow() = default; + + explicit GzipByteFlow(Gzip::Mode mode) { + gzip_.init(mode).ensure(); + } + + void init_decode() { + gzip_.init_decode().ensure(); + } + + void init_encode() { + gzip_.init_encode().ensure(); + } + + void set_max_output_size(size_t max_output_size) { + max_output_size_ = max_output_size; + } + + void loop() override; + + private: + Gzip gzip_; + size_t uncommited_size_ = 0; + size_t total_output_size_ = 0; + size_t max_output_size_ = std::numeric_limits::max(); + static constexpr size_t MIN_UPDATE_SIZE = 1 << 14; +}; +#endif + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h new file mode 100644 index 0000000000..e13dc8022e --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HazardPointers.h @@ -0,0 +1,133 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include +#include + +namespace td { + +template +class HazardPointers { + public: + explicit HazardPointers(size_t threads_n) : threads_(threads_n) { + for (auto &data : threads_) { + for (auto &ptr : data.hazard) { + ptr = nullptr; + } + } + } + HazardPointers(const HazardPointers &other) = delete; + HazardPointers &operator=(const HazardPointers &other) = delete; + HazardPointers(HazardPointers &&other) = delete; + HazardPointers &operator=(HazardPointers &&other) = delete; + + class Holder { + public: + T *protect(std::atomic &to_protect) { + return do_protect(hazard_ptr_, to_protect); + } + Holder(const Holder &other) = delete; + Holder &operator=(const Holder &other) = delete; + Holder(Holder &&other) = default; // TODO + Holder &operator=(Holder &&other) = delete; + ~Holder() { + clear(); + } + void clear() { + hazard_ptr_.store(nullptr, std::memory_order_release); + } + + private: + friend class HazardPointers; + explicit Holder(std::atomic &ptr) : hazard_ptr_(ptr) { + } + std::atomic &hazard_ptr_; + }; + + Holder get_holder(size_t thread_id, size_t pos) { + return Holder(get_hazard_ptr(thread_id, pos)); + } + + void retire(size_t thread_id, T *ptr = nullptr) { + CHECK(thread_id < threads_.size()); + auto &data = threads_[thread_id]; + if (ptr) { + data.to_delete.push_back(std::unique_ptr(ptr)); + } + for (auto it = data.to_delete.begin(); it != data.to_delete.end();) { + if (!is_protected(it->get())) { + it->reset(); + it = data.to_delete.erase(it); + } else { + ++it; + } + } + } + + // old inteface + T *protect(size_t thread_id, size_t pos, std::atomic &ptr) { + return do_protect(get_hazard_ptr(thread_id, pos), ptr); + } + void clear(size_t thread_id, size_t pos) { + do_clear(get_hazard_ptr(thread_id, pos)); + } + + size_t to_delete_size_unsafe() const { + size_t res = 0; + for (auto &thread : threads_) { + res += thread.to_delete.size(); + } + return res; + } + + private: + struct ThreadData { + std::array, MaxPointersN> hazard; + char pad[TD_CONCURRENCY_PAD - sizeof(std::array, MaxPointersN>)]; + + // stupid gc + std::vector> to_delete; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector>)]; + }; + std::vector threads_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector)]; + + static T *do_protect(std::atomic &hazard_ptr, std::atomic &to_protect) { + T *saved = nullptr; + T *to_save; + while ((to_save = to_protect.load()) != saved) { + hazard_ptr.store(to_save); + saved = to_save; + } + return saved; + } + + static void do_clear(std::atomic &hazard_ptr) { + hazard_ptr.store(nullptr, std::memory_order_release); + } + + bool is_protected(T *ptr) { + for (auto &thread : threads_) { + for (auto &hazard_ptr : thread.hazard) { + if (hazard_ptr.load() == ptr) { + return true; + } + } + } + return false; + } + + std::atomic &get_hazard_ptr(size_t thread_id, size_t pos) { + return threads_[thread_id].hazard[pos]; + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h new file mode 100644 index 0000000000..54ee391497 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Heap.h @@ -0,0 +1,152 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +namespace td { + +struct HeapNode { + bool in_heap() const { + return pos_ != -1; + } + bool is_top() const { + return pos_ == 0; + } + void remove() { + pos_ = -1; + } + int pos_ = -1; +}; + +template +class KHeap { + public: + bool empty() const { + return array_.empty(); + } + size_t size() const { + return array_.size(); + } + + KeyT top_key() const { + return array_[0].key_; + } + + HeapNode *pop() { + CHECK(!empty()); + HeapNode *result = array_[0].node_; + result->remove(); + erase(0); + return result; + } + + void insert(KeyT key, HeapNode *node) { + CHECK(!node->in_heap()); + array_.push_back({key, node}); + fix_up(static_cast(array_.size()) - 1); + } + + void fix(KeyT key, HeapNode *node) { + CHECK(node->in_heap()); + int pos = node->pos_; + KeyT old_key = array_[pos].key_; + array_[pos].key_ = key; + if (key < old_key) { + fix_up(pos); + } else { + fix_down(pos); + } + } + + void erase(HeapNode *node) { + CHECK(node->in_heap()); + int pos = node->pos_; + node->remove(); + erase(pos); + } + + template + void for_each(F &f) const { + for (auto &it : array_) { + f(it.key_, it.node_); + } + } + + void check() const { + for (size_t i = 0; i < array_.size(); i++) { + for (size_t j = i * K + 1; j < i * K + 1 + K && j < array_.size(); j++) { + CHECK(array_[i].key_ <= array_[j].key_) << i << " " << j; + } + } + } + + private: + struct Item { + KeyT key_; + HeapNode *node_; + }; + vector array_; + + void fix_up(int pos) { + auto item = array_[pos]; + + while (pos) { + int parent_pos = (pos - 1) / K; + auto parent_item = array_[parent_pos]; + + if (parent_item.key_ < item.key_) { + break; + } + + parent_item.node_->pos_ = pos; + array_[pos] = parent_item; + pos = parent_pos; + } + + item.node_->pos_ = pos; + array_[pos] = item; + } + + void fix_down(int pos) { + auto item = array_[pos]; + while (true) { + int left_pos = pos * K + 1; + int right_pos = min(left_pos + K, static_cast(array_.size())); + int next_pos = pos; + KeyT next_key = item.key_; + for (int i = left_pos; i < right_pos; i++) { + KeyT i_key = array_[i].key_; + if (i_key < next_key) { + next_key = i_key; + next_pos = i; + } + } + if (next_pos == pos) { + break; + } + array_[pos] = array_[next_pos]; + array_[pos].node_->pos_ = pos; + pos = next_pos; + } + + item.node_->pos_ = pos; + array_[pos] = item; + } + + void erase(int pos) { + array_[pos] = array_.back(); + array_.pop_back(); + if (pos < static_cast(array_.size())) { + fix_down(pos); + fix_up(pos); + } + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp new file mode 100644 index 0000000000..1e7449a668 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.cpp @@ -0,0 +1,191 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Hints.h" + +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/unicode.h" +#include "td/utils/utf8.h" + +#include + +namespace td { + +vector Hints::get_words(Slice name) { + bool in_word = false; + string word; + vector words; + auto pos = name.ubegin(); + auto end = name.uend(); + while (pos != end) { + uint32 code; + pos = next_utf8_unsafe(pos, &code); + + code = prepare_search_character(code); + if (code == 0) { + continue; + } + if (code == ' ') { + if (in_word) { + words.push_back(std::move(word)); + word.clear(); + in_word = false; + } + } else { + in_word = true; + append_utf8_character(word, code); + } + } + if (in_word) { + words.push_back(std::move(word)); + } + std::sort(words.begin(), words.end()); + + size_t new_words_size = 0; + for (size_t i = 0; i != words.size(); i++) { + if (i == words.size() - 1 || !begins_with(words[i + 1], words[i])) { + if (i != new_words_size) { + words[new_words_size] = std::move(words[i]); + } + // LOG(ERROR) << "Get word " << words[new_words_size]; + new_words_size++; + } + } + words.resize(new_words_size); + return words; +} + +void Hints::add(KeyT key, Slice name) { + // LOG(ERROR) << "Add " << key << ": " << name; + auto it = key_to_name_.find(key); + if (it != key_to_name_.end()) { + if (it->second == name) { + return; + } + auto old_words = get_words(it->second); + for (auto &old_word : old_words) { + vector &keys = word_to_keys_[old_word]; + auto key_it = std::find(keys.begin(), keys.end(), key); + CHECK(key_it != keys.end()); + if (keys.size() == 1) { + word_to_keys_.erase(old_word); + } else { + CHECK(keys.size() > 1); + *key_it = keys.back(); + keys.pop_back(); + } + } + } + if (name.empty()) { + if (it != key_to_name_.end()) { + key_to_name_.erase(it); + } + key_to_rating_.erase(key); + return; + } + auto words = get_words(name); + for (auto &word : words) { + vector &keys = word_to_keys_[word]; + CHECK(std::find(keys.begin(), keys.end(), key) == keys.end()); + keys.push_back(key); + } + key_to_name_[key] = name.str(); +} + +void Hints::set_rating(KeyT key, RatingT rating) { + // LOG(ERROR) << "Set rating " << key << ": " << rating; + key_to_rating_[key] = rating; +} + +vector Hints::search_word(const string &word) const { + // LOG(ERROR) << "Search word " << word; + vector results; + auto it = word_to_keys_.lower_bound(word); + while (it != word_to_keys_.end() && begins_with(it->first, word)) { + results.insert(results.end(), it->second.begin(), it->second.end()); + ++it; + } + + std::sort(results.begin(), results.end()); + results.erase(std::unique(results.begin(), results.end()), results.end()); + return results; +} + +std::pair> Hints::search(Slice query, int32 limit, bool return_all_for_empty_query) const { + // LOG(ERROR) << "Search " << query; + vector results; + + if (limit < 0) { + return {key_to_name_.size(), std::move(results)}; + } + + auto words = get_words(query); + if (return_all_for_empty_query && words.empty()) { + results.reserve(key_to_name_.size()); + for (auto &it : key_to_name_) { + results.push_back(it.first); + } + } + + for (size_t i = 0; i < words.size(); i++) { + vector keys = search_word(words[i]); + if (i == 0) { + results = std::move(keys); + continue; + } + + // now need to intersect two lists + size_t results_pos = 0; + size_t keys_pos = 0; + size_t new_results_size = 0; + while (results_pos != results.size() && keys_pos != keys.size()) { + if (results[results_pos] < keys[keys_pos]) { + results_pos++; + } else if (results[results_pos] > keys[keys_pos]) { + keys_pos++; + } else { + results[new_results_size++] = results[results_pos]; + results_pos++; + keys_pos++; + } + } + results.resize(new_results_size); + } + + auto total_size = results.size(); + if (total_size < static_cast(limit)) { + std::sort(results.begin(), results.end(), CompareByRating(key_to_rating_)); + } else { + std::partial_sort(results.begin(), results.begin() + limit, results.end(), CompareByRating(key_to_rating_)); + results.resize(limit); + } + + return {total_size, std::move(results)}; +} + +bool Hints::has_key(KeyT key) const { + return key_to_name_.find(key) != key_to_name_.end(); +} + +string Hints::key_to_string(KeyT key) const { + auto it = key_to_name_.find(key); + if (it == key_to_name_.end()) { + return string(); + } + return it->second; +} + +std::pair> Hints::search_empty(int32 limit) const { + return search(Slice(), limit, true); +} + +size_t Hints::size() const { + return key_to_name_.size(); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h new file mode 100644 index 0000000000..645896684a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Hints.h @@ -0,0 +1,76 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +#include +#include +#include + +namespace td { + +// TODO template KeyT +class Hints { + using KeyT = int64; + using RatingT = int64; + + public: + void add(KeyT key, Slice name); + + void remove(KeyT key) { + add(key, ""); + } + + void set_rating(KeyT key, RatingT rating); + + std::pair> search( + Slice query, int32 limit, + bool return_all_for_empty_query = false) const; // TODO sort by name instead of sort by rating + + bool has_key(KeyT key) const; + + string key_to_string(KeyT key) const; + + std::pair> search_empty(int32 limit) const; // == search("", limit, true) + + size_t size() const; + + private: + std::map> word_to_keys_; + std::unordered_map key_to_name_; + std::unordered_map key_to_rating_; + + static vector get_words(Slice name); + + vector search_word(const string &word) const; + + class CompareByRating { + const std::unordered_map &key_to_rating_; + + RatingT get_rating(const KeyT &key) const { + auto it = key_to_rating_.find(key); + if (it == key_to_rating_.end()) { + return RatingT(); + } + return it->second; + } + + public: + explicit CompareByRating(const std::unordered_map &key_to_rating) : key_to_rating_(key_to_rating) { + } + + bool operator()(const KeyT &lhs, const KeyT &rhs) const { + auto lhs_rating = get_rating(lhs); + auto rhs_rating = get_rating(rhs); + return lhs_rating < rhs_rating || (lhs_rating == rhs_rating && lhs < rhs); + } + }; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp new file mode 100644 index 0000000000..55b66f7b3a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.cpp @@ -0,0 +1,189 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/HttpUrl.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Parser.h" + +namespace td { + +string HttpUrl::get_url() const { + string result; + switch (protocol_) { + case Protocol::HTTP: + result += "http://"; + break; + case Protocol::HTTPS: + result += "https://"; + break; + default: + UNREACHABLE(); + } + if (!userinfo_.empty()) { + result += userinfo_; + result += '@'; + } + if (is_ipv6) { + result += '['; + } + result += host_; + if (is_ipv6) { + result += ']'; + } + if (specified_port_ > 0) { + result += ':'; + result += to_string(specified_port_); + } + CHECK(!query_.empty() && query_[0] == '/'); + result += query_; + return result; +} + +Result parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) { + // url == [https?://][userinfo@]host[:port] + Parser parser(url); + string protocol_str = to_lower(parser.read_till_nofail(':')); + + HttpUrl::Protocol protocol; + if (parser.start_with("://")) { + parser.advance(3); + if (protocol_str == "http") { + protocol = HttpUrl::Protocol::HTTP; + } else if (protocol_str == "https") { + protocol = HttpUrl::Protocol::HTTPS; + } else { + return Status::Error("Unsupported URL protocol"); + } + } else { + parser = Parser(url); + protocol = default_protocol; + } + Slice userinfo_host_port = parser.read_till_nofail("/?#"); + + int port = 0; + const char *colon = userinfo_host_port.end() - 1; + while (colon > userinfo_host_port.begin() && *colon != ':' && *colon != ']' && *colon != '@') { + colon--; + } + Slice userinfo_host; + if (colon > userinfo_host_port.begin() && *colon == ':') { + port = to_integer(Slice(colon + 1, userinfo_host_port.end())); + userinfo_host = Slice(userinfo_host_port.begin(), colon); + } else { + userinfo_host = userinfo_host_port; + } + if (port < 0 || port > 65535) { + return Status::Error("Wrong port number specified in the URL"); + } + + auto at_pos = userinfo_host.rfind('@'); + Slice userinfo = at_pos == static_cast(-1) ? "" : userinfo_host.substr(0, at_pos); + Slice host = userinfo_host.substr(at_pos + 1); + + bool is_ipv6 = false; + if (!host.empty() && host[0] == '[' && host.back() == ']') { + host.remove_prefix(1); + host.remove_suffix(1); + is_ipv6 = true; + } + if (host.empty()) { + return Status::Error("URL host is empty"); + } + + int specified_port = port; + if (port == 0) { + if (protocol == HttpUrl::Protocol::HTTP) { + port = 80; + } else { + CHECK(protocol == HttpUrl::Protocol::HTTPS); + port = 443; + } + } + + Slice query = parser.read_all(); + while (!query.empty() && is_space(query.back())) { + query.remove_suffix(1); + } + if (query.empty()) { + query = "/"; + } + string query_str; + if (query[0] != '/') { + query_str = '/'; + } + for (auto c : query) { + if (static_cast(c) <= 0x20) { + query_str += '%'; + query_str += "0123456789ABCDEF"[c / 16]; + query_str += "0123456789ABCDEF"[c % 16]; + } else { + query_str += c; + } + } + + string host_str = to_lower(host); + for (size_t i = 0; i < host_str.size(); i++) { + char c = host_str[i]; + if (('a' <= c && c <= 'z') || c == '.' || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '!' || c == '$' || + c == ',' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')' || c == ';' || c == '&' || c == '+' || + c == '=') { + // symbols allowed by RFC 7230 and RFC 3986 + continue; + } + if (c == '%') { + c = host_str[++i]; + if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) { + c = host_str[++i]; + if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) { + // percent encoded symbol as allowed by RFC 7230 and RFC 3986 + continue; + } + } + } + // all other symbols aren't allowed + unsigned char uc = static_cast(c); + if (uc >= 128) { + // but we allow plain UTF-8 symbols + continue; + } + return Status::Error("Wrong URL host"); + } + + return HttpUrl{protocol, userinfo.str(), std::move(host_str), is_ipv6, specified_port, port, std::move(query_str)}; +} + +StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) { + sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::HTTP ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_) + << tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_); + return sb; +} + +string get_url_query_file_name(const string &query) { + Slice query_slice = query; + query_slice.truncate(query.find_first_of("?#")); + + auto slash_pos = query_slice.rfind('/'); + if (slash_pos < query_slice.size()) { + return query_slice.substr(slash_pos + 1).str(); + } + return query_slice.str(); +} + +string get_url_file_name(const string &url) { + // TODO remove copy + string url_copy = url; + auto r_http_url = parse_url(url_copy); + if (r_http_url.is_error()) { + LOG(WARNING) << "Receive wrong URL \"" << url << '"'; + return string(); + } + return get_url_query_file_name(r_http_url.ok().query_); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h new file mode 100644 index 0000000000..f7d1e4aaba --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/HttpUrl.h @@ -0,0 +1,39 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +namespace td { + +class HttpUrl { + public: + enum class Protocol { HTTP, HTTPS } protocol_; + string userinfo_; + string host_; + bool is_ipv6; + int specified_port_; + int port_; + string query_; + + string get_url() const; +}; + +// TODO Slice instead of MutableSlice +Result parse_url(MutableSlice url, + HttpUrl::Protocol default_protocol = HttpUrl::Protocol::HTTP) TD_WARN_UNUSED_RESULT; + +StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url); + +string get_url_query_file_name(const string &query); + +string get_url_file_name(const string &url); + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp new file mode 100644 index 0000000000..eb654f43cd --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.cpp @@ -0,0 +1,648 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/JsonBuilder.h" + +#include "td/utils/misc.h" +#include "td/utils/ScopeGuard.h" + +#include + +namespace td { +StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val) { + sb << '"'; + SCOPE_EXIT { + sb << '"'; + }; + auto *s = val.value_.begin(); + auto len = val.value_.size(); + + for (size_t pos = 0; pos < len; pos++) { + auto ch = static_cast(s[pos]); + switch (ch) { + case '"': + sb << '\\' << '"'; + break; + case '\\': + sb << '\\' << '\\'; + break; + case '\b': + sb << '\\' << 'b'; + break; + case '\f': + sb << '\\' << 'f'; + break; + case '\n': + sb << '\\' << 'n'; + break; + case '\r': + sb << '\\' << 'r'; + break; + case '\t': + sb << '\\' << 't'; + break; + default: + if (ch <= 31) { + sb << JsonOneChar(s[pos]); + break; + } + sb << s[pos]; + break; + } + } + return sb; +} + +StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) { + sb << '"'; + SCOPE_EXIT { + sb << '"'; + }; + auto *s = val.str_.begin(); + auto len = val.str_.size(); + + for (size_t pos = 0; pos < len; pos++) { + auto ch = static_cast(s[pos]); + switch (ch) { + case '"': + sb << '\\' << '"'; + break; + case '\\': + sb << '\\' << '\\'; + break; + case '\b': + sb << '\\' << 'b'; + break; + case '\f': + sb << '\\' << 'f'; + break; + case '\n': + sb << '\\' << 'n'; + break; + case '\r': + sb << '\\' << 'r'; + break; + case '\t': + sb << '\\' << 't'; + break; + default: + if (ch <= 31) { + sb << JsonOneChar(s[pos]); + break; + } + if (128 <= ch) { + int a = s[pos]; + CHECK((a & 0x40) != 0); + + CHECK(pos + 1 < len); + int b = s[++pos]; + CHECK((b & 0xc0) == 0x80); + if ((a & 0x20) == 0) { + CHECK((a & 0x1e) > 0); + sb << JsonChar(((a & 0x1f) << 6) | (b & 0x3f)); + break; + } + + CHECK(pos + 1 < len); + int c = s[++pos]; + CHECK((c & 0xc0) == 0x80); + if ((a & 0x10) == 0) { + CHECK(((a & 0x0f) | (b & 0x20)) > 0); + sb << JsonChar(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + break; + } + + CHECK(pos + 1 < len); + int d = s[++pos]; + CHECK((d & 0xc0) == 0x80); + if ((a & 0x08) == 0) { + CHECK(((a & 0x07) | (b & 0x30)) > 0); + sb << JsonChar(((a & 0x07) << 18) | ((b & 0x3f) << 12) | ((c & 0x3f) << 6) | (d & 0x3f)); + break; + } + + UNREACHABLE(); + break; + } + sb << s[pos]; + break; + } + } + return sb; +} +Result json_string_decode(Parser &parser) { + if (!parser.try_skip('"')) { + return Status::Error("Opening '\"' expected"); + } + auto *cur_src = parser.data().data(); + auto *end_src = parser.data().end(); + auto *end = cur_src; + while (end < end_src && end[0] != '"') { + if (end[0] == '\\') { + end++; + } + end++; + } + if (end >= end_src) { + return Status::Error("Closing '\"' not found"); + } + parser.advance(end + 1 - cur_src); + end_src = end; + + auto *cur_dest = cur_src; + auto *begin_dest = cur_src; + + while (cur_src != end_src) { + auto *slash = static_cast(std::memchr(cur_src, '\\', end_src - cur_src)); + if (slash == nullptr) { + slash = end_src; + } + std::memmove(cur_dest, cur_src, slash - cur_src); + cur_dest += slash - cur_src; + cur_src = slash; + if (cur_src != end_src) { + cur_src++; + if (cur_src == end_src) { + // TODO UNREACHABLE(); + return Status::Error("Unexpected end of string"); + } + switch (*cur_src) { + case '"': + case '\\': + case '/': + *cur_dest++ = *cur_src++; + break; + case 'b': + *cur_dest++ = '\b'; + cur_src++; + break; + case 'f': + *cur_dest++ = '\f'; + cur_src++; + break; + case 'n': + *cur_dest++ = '\n'; + cur_src++; + break; + case 'r': + *cur_dest++ = '\r'; + cur_src++; + break; + case 't': + *cur_dest++ = '\t'; + cur_src++; + break; + case 'u': { + cur_src++; + if (cur_src + 4 > end_src) { + return Status::Error("\\u has less than 4 symbols"); + } + int num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + num = num * 16 + d; + } + if (0xD7FF < num && num < 0xE000) { + if (cur_src + 6 <= end_src && cur_src[0] == '\\' && cur_src[1] == 'u') { + cur_src += 2; + int new_num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + new_num = new_num * 16 + d; + } + if (0xD7FF < new_num && new_num < 0xE000) { + num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000; + } else { + cur_src -= 6; + } + } + } + + if (num < 128) { + *cur_dest++ = static_cast(num); + } else if (num < 0x800) { + *cur_dest++ = static_cast(0xc0 + (num >> 6)); + *cur_dest++ = static_cast(0x80 + (num & 63)); + } else if (num < 0xffff) { + *cur_dest++ = static_cast(0xe0 + (num >> 12)); + *cur_dest++ = static_cast(0x80 + ((num >> 6) & 63)); + *cur_dest++ = static_cast(0x80 + (num & 63)); + } else { + *cur_dest++ = static_cast(0xf0 + (num >> 18)); + *cur_dest++ = static_cast(0x80 + ((num >> 12) & 63)); + *cur_dest++ = static_cast(0x80 + ((num >> 6) & 63)); + *cur_dest++ = static_cast(0x80 + (num & 63)); + } + break; + } + } + } + } + CHECK(cur_dest <= end_src); + return MutableSlice(begin_dest, cur_dest); +} + +Status json_string_skip(Parser &parser) { + if (!parser.try_skip('"')) { + return Status::Error("Opening '\"' expected"); + } + auto *begin_src = parser.data().data(); + auto *cur_src = begin_src; + auto *end_src = parser.data().end(); + auto *end = cur_src; + while (end < end_src && *end != '"') { + if (*end == '\\') { + end++; + } + end++; + } + if (end >= end_src) { + return Status::Error("Closing '\"' not found"); + } + parser.advance(end + 1 - cur_src); + end_src = end; + + while (cur_src != end_src) { + auto *slash = static_cast(std::memchr(cur_src, '\\', end_src - cur_src)); + if (slash == nullptr) { + slash = end_src; + } + cur_src = slash; + if (cur_src != end_src) { + cur_src++; + if (cur_src == end_src) { + // TODO UNREACHABLE(); + return Status::Error("Unexpected end of string"); + } + switch (*cur_src) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + cur_src++; + break; + case 'u': { + cur_src++; + if (cur_src + 4 > end_src) { + return Status::Error("\\u has less than 4 symbols"); + } + int num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + num = num * 16 + d; + } + if (0xD7FF < num && num < 0xE000) { + if (cur_src + 6 <= end_src && cur_src[0] == '\\' && cur_src[1] == 'u') { + cur_src += 2; + int new_num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + new_num = new_num * 16 + d; + } + if (0xD7FF < new_num && new_num < 0xE000) { + // num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000; + } else { + cur_src -= 6; + } + } + } + break; + } + } + } + } + return Status::OK(); +} + +Result do_json_decode(Parser &parser, int32 max_depth) { + if (max_depth < 0) { + return Status::Error("Too big object depth"); + } + + parser.skip_whitespaces(); + switch (parser.peek_char()) { + case 'f': + if (parser.skip_start_with("false")) { + return JsonValue::create_boolean(false); + } + return Status::Error("Starts with 'f' -- false expected"); + case 't': + if (parser.skip_start_with("true")) { + return JsonValue::create_boolean(true); + } + return Status::Error("Starts with 't' -- true expected"); + case 'n': + if (parser.skip_start_with("null")) { + return JsonValue(); + } + return Status::Error("Starts with 'n' -- null expected"); + case '"': { + TRY_RESULT(slice, json_string_decode(parser)); + return JsonValue::create_string(slice); + } + case '[': { + parser.skip('['); + parser.skip_whitespaces(); + std::vector res; + if (parser.try_skip(']')) { + return JsonValue::create_array(std::move(res)); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_RESULT(value, do_json_decode(parser, max_depth - 1)); + res.emplace_back(std::move(value)); + + parser.skip_whitespaces(); + if (parser.try_skip(']')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return JsonValue::create_array(std::move(res)); + } + case '{': { + parser.skip('{'); + parser.skip_whitespaces(); + std::vector > res; + if (parser.try_skip('}')) { + return JsonValue::make_object(std::move(res)); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_RESULT(key, json_string_decode(parser)); + parser.skip_whitespaces(); + if (!parser.try_skip(':')) { + return Status::Error("':' expected"); + } + TRY_RESULT(value, do_json_decode(parser, max_depth - 1)); + res.emplace_back(std::move(key), std::move(value)); + + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return JsonValue::make_object(std::move(res)); + } + case '-': + case '+': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + auto num = parser.read_while( + [](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; }); + return JsonValue::create_number(num); + } + case 0: + return Status::Error("Unexpected end"); + default: { + char next = parser.peek_char(); + if (0 < next && next < 127) { + return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'"); + } else { + return Status::Error("Unexpected symbol"); + } + } + } + UNREACHABLE(); +} + +Status do_json_skip(Parser &parser, int32 max_depth) { + if (max_depth < 0) { + return Status::Error("Too big object depth"); + } + + parser.skip_whitespaces(); + switch (parser.peek_char()) { + case 'f': + if (parser.skip_start_with("false")) { + return Status::OK(); + } + return Status::Error("Starts with 'f' -- false expected"); + case 't': + if (parser.skip_start_with("true")) { + return Status::OK(); + } + return Status::Error("Starts with 't' -- true expected"); + case 'n': + if (parser.skip_start_with("null")) { + return Status::OK(); + } + return Status::Error("Starts with 'n' -- null expected"); + case '"': { + return json_string_skip(parser); + } + case '[': { + parser.skip('['); + parser.skip_whitespaces(); + if (parser.try_skip(']')) { + return Status::OK(); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_STATUS(do_json_skip(parser, max_depth - 1)); + + parser.skip_whitespaces(); + if (parser.try_skip(']')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return Status::OK(); + } + case '{': { + parser.skip('{'); + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + return Status::OK(); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_STATUS(json_string_skip(parser)); + parser.skip_whitespaces(); + if (!parser.try_skip(':')) { + return Status::Error("':' expected"); + } + TRY_STATUS(do_json_skip(parser, max_depth - 1)); + + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return Status::OK(); + } + case '-': + case '+': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + parser.read_while( + [](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; }); + return Status::OK(); + } + case 0: + return Status::Error("Unexpected end"); + default: { + char next = parser.peek_char(); + if (0 < next && next < 127) { + return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'"); + } else { + return Status::Error("Unexpected symbol"); + } + } + } + return Status::Error("Can't parse"); +} + +Slice JsonValue::get_type_name(Type type) { + switch (type) { + case Type::Null: + return Slice("Null"); + case Type::Number: + return Slice("Number"); + case Type::Boolean: + return Slice("Boolean"); + case Type::String: + return Slice("String"); + case Type::Array: + return Slice("Array"); + case Type::Object: + return Slice("Object"); + default: + UNREACHABLE(); + return Slice("Unknown"); + } +} + +bool has_json_object_field(JsonObject &object, Slice name) { + for (auto &field_value : object) { + if (field_value.first == name) { + return true; + } + } + return false; +} + +Result get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, bool is_optional) { + for (auto &field_value : object) { + if (field_value.first == name) { + if (type != JsonValue::Type::Null && field_value.second.type() != type) { + return Status::Error(400, PSLICE() + << "Field \"" << name << "\" must be of type " << JsonValue::get_type_name(type)); + } + + return std::move(field_value.second); + } + } + if (!is_optional) { + return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\""); + } + return JsonValue(); +} + +Result get_json_object_bool_field(JsonObject &object, Slice name, bool is_optional, bool default_value) { + TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Boolean, is_optional)); + if (value.type() == JsonValue::Type::Null) { + return default_value; + } + return value.get_boolean(); +} + +Result get_json_object_int_field(JsonObject &object, Slice name, bool is_optional, int32 default_value) { + TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Number, is_optional)); + if (value.type() == JsonValue::Type::Null) { + return default_value; + } + return to_integer_safe(value.get_number()); +} + +Result get_json_object_double_field(JsonObject &object, Slice name, bool is_optional, double default_value) { + TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Number, is_optional)); + if (value.type() == JsonValue::Type::Null) { + return default_value; + } + return to_double(value.get_number()); +} + +Result get_json_object_string_field(JsonObject &object, Slice name, bool is_optional, string default_value) { + for (auto &field_value : object) { + if (field_value.first == name) { + if (field_value.second.type() == JsonValue::Type::String) { + return field_value.second.get_string().str(); + } + if (field_value.second.type() == JsonValue::Type::Number) { + return field_value.second.get_number().str(); + } + + return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type String"); + } + } + if (is_optional) { + return default_value; + } + return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\""); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h new file mode 100644 index 0000000000..735c4b29ec --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/JsonBuilder.h @@ -0,0 +1,760 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Parser.h" +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include +#include +#include +#include + +namespace td { + +template +std::tuple ctie(const Args &... args) TD_WARN_UNUSED_RESULT; + +template +std::tuple ctie(const Args &... args) { + return std::tie(args...); +} + +class JsonTrue { + public: + friend StringBuilder &operator<<(StringBuilder &sb, const JsonTrue &val) { + return sb << "true"; + } +}; + +class JsonFalse { + public: + friend StringBuilder &operator<<(StringBuilder &sb, const JsonFalse &val) { + return sb << "false"; + } +}; + +class JsonNull { + public: + friend StringBuilder &operator<<(StringBuilder &sb, JsonNull val) { + return sb << "null"; + } +}; + +class JsonBool { + public: + explicit JsonBool(bool value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonBool &val) { + if (val.value_) { + return sb << JsonTrue(); + } else { + return sb << JsonFalse(); + } + } + + private: + bool value_; +}; + +class JsonInt { + public: + explicit JsonInt(int32 value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonInt &val) { + return sb << val.value_; + } + + private: + int32 value_; +}; + +class JsonLong { + public: + explicit JsonLong(int64 value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonLong &val) { + return sb << val.value_; + } + + private: + int64 value_; +}; + +class JsonFloat { + public: + explicit JsonFloat(double value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonFloat &val) { + return sb << val.value_; + } + + private: + double value_; +}; + +class JsonOneChar { + public: + explicit JsonOneChar(unsigned int c) : c_(c) { + } + + friend StringBuilder &operator<<(StringBuilder &sb, const JsonOneChar &val) { + auto c = val.c_; + return sb << '\\' << 'u' << "0123456789abcdef"[c >> 12] << "0123456789abcdef"[(c >> 8) & 15] + << "0123456789abcdef"[(c >> 4) & 15] << "0123456789abcdef"[c & 15]; + } + + private: + unsigned int c_; +}; + +class JsonChar { + public: + explicit JsonChar(unsigned int c) : c_(c) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonChar &val) { + auto c = val.c_; + if (c < 0x10000) { + if (0xD7FF < c && c < 0xE000) { + // UTF-8 correctness has already been checked + UNREACHABLE(); + } + return sb << JsonOneChar(c); + } else if (c <= 0x10ffff) { + return sb << JsonOneChar(0xD7C0 + (c >> 10)) << JsonOneChar(0xDC00 + (c & 0x3FF)); + } else { + // UTF-8 correctness has already been checked + UNREACHABLE(); + } + } + + private: + unsigned int c_; +}; + +class JsonRaw { + public: + explicit JsonRaw(Slice value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonRaw &val) { + return sb << val.value_; + } + + private: + Slice value_; +}; + +class JsonRawString { + public: + explicit JsonRawString(Slice value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val); + + private: + Slice value_; +}; + +class JsonString { + public: + explicit JsonString(Slice str) : str_(str) { + } + + friend StringBuilder &operator<<(StringBuilder &sb, const JsonString &val); + + private: + Slice str_; +}; + +class JsonScope; +class JsonValueScope; +class JsonArrayScope; +class JsonObjectScope; + +class JsonBuilder { + public: + explicit JsonBuilder(StringBuilder &&sb) : sb_(std::move(sb)) { + } + StringBuilder &string_builder() { + return sb_; + } + friend class JsonScope; + JsonValueScope enter_value() TD_WARN_UNUSED_RESULT; + JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT; + JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT; + + private: + StringBuilder sb_; + JsonScope *scope_ = nullptr; +}; + +class Jsonable {}; + +class JsonScope { + public: + explicit JsonScope(JsonBuilder *jb) : sb_(&jb->sb_), jb_(jb) { + save_scope_ = jb_->scope_; + jb_->scope_ = this; + CHECK(is_active()); + } + JsonScope(const JsonScope &other) = delete; + JsonScope(JsonScope &&other) : sb_(other.sb_), jb_(other.jb_), save_scope_(other.save_scope_) { + other.jb_ = nullptr; + } + JsonScope &operator=(const JsonScope &) = delete; + JsonScope &operator=(JsonScope &&) = delete; + ~JsonScope() { + if (jb_) { + leave(); + } + } + void leave() { + CHECK(is_active()); + jb_->scope_ = save_scope_; + } + + protected: + StringBuilder *sb_; + + // For CHECK + JsonBuilder *jb_; + JsonScope *save_scope_; + + bool is_active() const { + return jb_ && jb_->scope_ == this; + } + + JsonScope &operator<<(JsonTrue x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(JsonFalse x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(JsonNull x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonBool &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonInt &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonLong &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonFloat &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonString &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonRawString &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonRaw &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(bool x) { + return *this << JsonBool(x); + } + JsonScope &operator<<(int32 x) { + return *this << JsonInt(x); + } + JsonScope &operator<<(int64 x) { + return *this << JsonLong(x); + } + JsonScope &operator<<(double x) { + return *this << JsonFloat(x); + } + template + JsonScope &operator<<(const T *x); // not implemented + template + JsonScope &operator<<(const char (&x)[N]) { + return *this << JsonString(Slice(x)); + } + JsonScope &operator<<(const char *x) { + return *this << JsonString(Slice(x)); + } + JsonScope &operator<<(const string &x) { + return *this << JsonString(Slice(x)); + } + JsonScope &operator<<(Slice x) { + return *this << JsonString(x); + } +}; + +class JsonValueScope : public JsonScope { + public: + using JsonScope::JsonScope; + template + std::enable_if_t::type>::value, JsonValueScope &> operator<<( + const T &x) { + x.store(this); + return *this; + } + template + std::enable_if_t::type>::value, JsonValueScope &> operator<<( + const T &x) { + CHECK(!was_); + was_ = true; + JsonScope::operator<<(x); + return *this; + } + + JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT; + JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT; + + private: + bool was_ = false; +}; + +class JsonArrayScope : public JsonScope { + public: + explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) { + *sb_ << "["; + } + JsonArrayScope(JsonArrayScope &&other) = default; + ~JsonArrayScope() { + if (jb_) { + leave(); + } + } + void leave() { + *sb_ << "]"; + } + template + JsonArrayScope &operator<<(const T &x) { + enter_value() << x; + return *this; + } + JsonValueScope enter_value() { + CHECK(is_active()); + if (is_first_) { + *sb_ << ","; + } else { + is_first_ = true; + } + return jb_->enter_value(); + } + + private: + bool is_first_ = false; +}; + +class JsonObjectScope : public JsonScope { + public: + explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) { + *sb_ << "{"; + } + JsonObjectScope(JsonObjectScope &&other) = default; + ~JsonObjectScope() { + if (jb_) { + leave(); + } + } + void leave() { + *sb_ << "}"; + } + template + JsonObjectScope &operator<<(std::tuple key_value) { + return *this << std::pair(std::get<0>(key_value), std::get<1>(key_value)); + } + template + JsonObjectScope &operator<<(std::pair key_value) { + CHECK(is_active()); + if (is_first_) { + *sb_ << ","; + } else { + is_first_ = true; + } + jb_->enter_value() << key_value.first; + *sb_ << ":"; + jb_->enter_value() << key_value.second; + return *this; + } + JsonObjectScope &operator<<(const JsonRaw &key_value) { + CHECK(is_active()); + is_first_ = true; + jb_->enter_value() << key_value; + return *this; + } + + private: + bool is_first_ = false; +}; + +inline JsonArrayScope JsonValueScope::enter_array() { + CHECK(!was_); + was_ = true; + return JsonArrayScope(jb_); +} +inline JsonObjectScope JsonValueScope::enter_object() { + CHECK(!was_); + was_ = true; + return JsonObjectScope(jb_); +} +inline JsonValueScope JsonBuilder::enter_value() { + return JsonValueScope(this); +} +inline JsonObjectScope JsonBuilder::enter_object() { + return JsonObjectScope(this); +} +inline JsonArrayScope JsonBuilder::enter_array() { + return JsonArrayScope(this); +} + +class JsonValue; + +using JsonObject = vector>; +using JsonArray = vector; + +class JsonValue : public Jsonable { + public: + enum class Type { Null, Number, Boolean, String, Array, Object }; + + static Slice get_type_name(Type type); + + JsonValue() { + } + ~JsonValue() { + destroy(); + } + JsonValue(JsonValue &&other) : JsonValue() { + init(std::move(other)); + } + JsonValue &operator=(JsonValue &&other) { + if (&other == this) { + return *this; + } + destroy(); + init(std::move(other)); + return *this; + } + JsonValue(const JsonValue &other) = delete; + JsonValue &operator=(const JsonValue &other) = delete; + + Type type() const { + return type_; + } + + MutableSlice &get_string() { + CHECK(type_ == Type::String); + return string_; + } + const MutableSlice &get_string() const { + CHECK(type_ == Type::String); + return string_; + } + bool &get_boolean() { + CHECK(type_ == Type::Boolean); + return boolean_; + } + const bool &get_boolean() const { + CHECK(type_ == Type::Boolean); + return boolean_; + } + + MutableSlice &get_number() { + CHECK(type_ == Type::Number); + return number_; + } + const MutableSlice &get_number() const { + CHECK(type_ == Type::Number); + return number_; + } + + JsonArray &get_array() { + CHECK(type_ == Type::Array); + return array_; + } + const JsonArray &get_array() const { + CHECK(type_ == Type::Array); + return array_; + } + + JsonObject &get_object() { + CHECK(type_ == Type::Object); + return object_; + } + const JsonObject &get_object() const { + CHECK(type_ == Type::Object); + return object_; + } + + static JsonValue create_boolean(bool val) { + JsonValue res; + res.init_boolean(val); + return res; + } + + static JsonValue create_number(MutableSlice number) { + JsonValue res; + res.init_number(number); + return res; + } + + static JsonValue create_string(MutableSlice str) { + JsonValue res; + res.init_string(str); + return res; + } + + static JsonValue create_array(JsonArray v) { + JsonValue res; + res.init_array(std::move(v)); + return res; + } + + static JsonValue make_object(JsonObject c) { + JsonValue res; + res.init_object(std::move(c)); + return res; + } + + void store(JsonValueScope *scope) const { + switch (type_) { + case Type::Null: + *scope << JsonRaw("null"); + break; + case Type::Boolean: + if (get_boolean()) { + *scope << JsonRaw("true"); + } else { + *scope << JsonRaw("false"); + } + break; + case Type::Number: + *scope << JsonRaw(get_number()); + break; + case Type::String: + *scope << JsonString(get_string()); + break; + case Type::Array: { + auto arr = scope->enter_array(); + for (auto &val : get_array()) { + arr << val; + } + break; + } + case Type::Object: { + auto object = scope->enter_object(); + for (auto &key_value : get_object()) { + object << ctie(JsonString(key_value.first), key_value.second); + } + break; + } + } + }; + + private: + Type type_{Type::Null}; + union { + MutableSlice number_; + bool boolean_; + MutableSlice string_; + JsonArray array_; + JsonObject object_; + }; + + void init_null() { + type_ = Type::Null; + } + void init_number(MutableSlice number) { + type_ = Type::Number; + new (&number_) MutableSlice(number); + } + void init_boolean(bool boolean) { + type_ = Type::Boolean; + boolean_ = boolean; + } + void init_string(MutableSlice slice) { + type_ = Type::String; + new (&string_) MutableSlice(slice); + } + void init_array(JsonArray array) { + type_ = Type::Array; + new (&array_) JsonArray(std::move(array)); + } + void init_object(JsonObject object) { + type_ = Type::Object; + new (&object_) JsonObject(std::move(object)); + } + + void init(JsonValue &&other) { + switch (other.type_) { + case Type::Null: + break; + case Type::Number: + init_number(other.number_); + break; + case Type::Boolean: + init_boolean(other.boolean_); + break; + case Type::String: + init_string(other.string_); + break; + case Type::Array: + init_array(std::move(other.array_)); + break; + case Type::Object: + init_object(std::move(other.object_)); + break; + } + other.destroy(); + } + + void destroy() { + switch (type_) { + case Type::Null: + case Type::Boolean: + break; + case Type::Number: + number_.~MutableSlice(); + break; + case Type::String: + string_.~MutableSlice(); + break; + case Type::Array: + array_.~vector(); + break; + case Type::Object: + object_.~vector>(); + break; + } + type_ = Type::Null; + } +}; + +inline StringBuilder &operator<<(StringBuilder &sb, JsonValue::Type type) { + switch (type) { + case JsonValue::Type::Object: + return sb << "JsonObject"; + case JsonValue::Type::Boolean: + return sb << "JsonBoolean"; + case JsonValue::Type::Null: + return sb << "JsonNull"; + case JsonValue::Type::Number: + return sb << "JsonNumber"; + case JsonValue::Type::Array: + return sb << "JsonArray"; + case JsonValue::Type::String: + return sb << "JsonString"; + default: + UNREACHABLE(); + return sb; + } +} + +class VirtuallyJsonable : public Jsonable { + public: + virtual void store(JsonValueScope *scope) const = 0; + VirtuallyJsonable() = default; + VirtuallyJsonable(const VirtuallyJsonable &) = delete; + VirtuallyJsonable &operator=(const VirtuallyJsonable &) = delete; + VirtuallyJsonable(VirtuallyJsonable &&) = default; + VirtuallyJsonable &operator=(VirtuallyJsonable &&) = default; + virtual ~VirtuallyJsonable() = default; +}; + +class VirtuallyJsonableInt : public VirtuallyJsonable { + public: + explicit VirtuallyJsonableInt(int32 value) : value_(value) { + } + void store(JsonValueScope *scope) const override { + *scope << JsonInt(value_); + } + + private: + int32 value_; +}; + +class VirtuallyJsonableLong : public VirtuallyJsonable { + public: + explicit VirtuallyJsonableLong(int64 value) : value_(value) { + } + void store(JsonValueScope *scope) const override { + *scope << JsonLong(value_); + } + + private: + int64 value_; +}; + +class VirtuallyJsonableString : public VirtuallyJsonable { + public: + explicit VirtuallyJsonableString(Slice value) : value_(value) { + } + void store(JsonValueScope *scope) const override { + *scope << JsonString(value_); + } + + private: + Slice value_; +}; + +Result json_string_decode(Parser &parser) TD_WARN_UNUSED_RESULT; +Status json_string_skip(Parser &parser) TD_WARN_UNUSED_RESULT; + +Result do_json_decode(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT; +Status do_json_skip(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT; + +inline Result json_decode(MutableSlice from) { + Parser parser(from); + const int32 DEFAULT_MAX_DEPTH = 100; + auto result = do_json_decode(parser, DEFAULT_MAX_DEPTH); + if (result.is_ok()) { + parser.skip_whitespaces(); + if (!parser.empty()) { + return Status::Error("Expected string end"); + } + } + return result; +} + +template +StrT json_encode(const ValT &val) { + auto buf_len = 1 << 19; + auto buf = StackAllocator::alloc(buf_len); + JsonBuilder jb(StringBuilder(buf.as_slice())); + jb.enter_value() << val; + LOG_IF(ERROR, jb.string_builder().is_error()) << "Json buffer overflow"; + auto slice = jb.string_builder().as_cslice(); + return StrT(slice.begin(), slice.size()); +} + +bool has_json_object_field(JsonObject &object, Slice name); + +Result get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, + bool is_optional = true) TD_WARN_UNUSED_RESULT; + +Result get_json_object_bool_field(JsonObject &object, Slice name, bool is_optional = true, + bool default_value = false) TD_WARN_UNUSED_RESULT; + +Result get_json_object_int_field(JsonObject &object, Slice name, bool is_optional = true, + int32 default_value = 0) TD_WARN_UNUSED_RESULT; + +Result get_json_object_double_field(JsonObject &object, Slice name, bool is_optional = true, + double default_value = 0.0) TD_WARN_UNUSED_RESULT; + +Result get_json_object_string_field(JsonObject &object, Slice name, bool is_optional = true, + string default_value = "") TD_WARN_UNUSED_RESULT; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h new file mode 100644 index 0000000000..1606c44d2b --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/List.h @@ -0,0 +1,92 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/logging.h" + +namespace td { + +struct ListNode { + ListNode *next; + ListNode *prev; + ListNode() { + clear(); + } + + ~ListNode() { + remove(); + } + + ListNode(const ListNode &) = delete; + ListNode &operator=(const ListNode &) = delete; + + ListNode(ListNode &&other) { + if (other.empty()) { + clear(); + } else { + ListNode *head = other.prev; + other.remove(); + head->put(this); + } + } + + ListNode &operator=(ListNode &&other) { + this->remove(); + + if (!other.empty()) { + ListNode *head = other.prev; + other.remove(); + head->put(this); + } + + return *this; + } + + void connect(ListNode *to) { + CHECK(to != nullptr); + next = to; + to->prev = this; + } + + void remove() { + prev->connect(next); + clear(); + } + + void put(ListNode *other) { + other->connect(next); + this->connect(other); + } + + void put_back(ListNode *other) { + prev->connect(other); + other->connect(this); + } + + ListNode *get() { + ListNode *result = prev; + if (result == this) { + return nullptr; + } + result->prev->connect(this); + result->clear(); + // this->connect(result->next); + return result; + } + + bool empty() const { + return next == this; + } + + private: + void clear() { + next = this; + prev = this; + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h new file mode 100644 index 0000000000..aa125df2f7 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MemoryLog.h @@ -0,0 +1,83 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +#include +#include +#include + +namespace td { + +template +class MemoryLog : public LogInterface { + static constexpr size_t MAX_OUTPUT_SIZE = buffer_size / 16 < (8 << 10) ? buffer_size / 16 : (8 << 10); + + public: + MemoryLog() { + std::memset(buffer_, ' ', sizeof(buffer_)); + } + + void append(CSlice new_slice, int log_level) override { + Slice slice = new_slice; + slice.truncate(MAX_OUTPUT_SIZE); + while (!slice.empty() && slice.back() == '\n') { + slice.remove_suffix(1); + } + size_t slice_size = slice.size(); + CHECK(slice_size * 3 < buffer_size); + size_t pad_size = ((slice_size + 15) & ~15) - slice_size; + constexpr size_t magic_size = 16; + uint32 total_size = static_cast(slice_size + pad_size + magic_size); + uint32 real_pos = pos_.fetch_add(total_size, std::memory_order_relaxed); + CHECK((total_size & 15) == 0); + + uint32 start_pos = real_pos & (buffer_size - 1); + uint32 end_pos = start_pos + total_size; + if (likely(end_pos <= buffer_size)) { + std::memcpy(&buffer_[start_pos + magic_size], slice.data(), slice_size); + std::memcpy(&buffer_[start_pos + magic_size + slice_size], " ", pad_size); + } else { + size_t first = buffer_size - start_pos - magic_size; + size_t second = slice_size - first; + std::memcpy(&buffer_[start_pos + magic_size], slice.data(), first); + std::memcpy(&buffer_[0], slice.data() + first, second); + std::memcpy(&buffer_[second], " ", pad_size); + } + + CHECK((start_pos & 15) == 0); + CHECK(start_pos <= buffer_size - magic_size); + buffer_[start_pos] = '\n'; + size_t printed = std::snprintf(&buffer_[start_pos + 1], magic_size - 1, "LOG:%08x: ", real_pos); + CHECK(printed == magic_size - 2); + buffer_[start_pos + magic_size - 1] = ' '; + + if (log_level == VERBOSITY_NAME(FATAL)) { + process_fatal_error(new_slice); + } + } + + void rotate() override { + } + + Slice get_buffer() const { + return Slice(buffer_, sizeof(buffer_)); + } + + size_t get_pos() const { + return pos_ & (buffer_size - 1); + } + + private: + char buffer_[buffer_size]; + std::atomic pos_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp new file mode 100644 index 0000000000..75c4fe34b5 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.cpp @@ -0,0 +1,44 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/MimeType.h" + +#include "td/utils/logging.h" + +const char *extension_to_mime_type(const char *extension, size_t extension_len); // auto-generated +const char *mime_type_to_extension(const char *mime_type, size_t mime_type_len); // auto-generated + +namespace td { + +string MimeType::to_extension(Slice mime_type, Slice default_value) { + if (mime_type.empty()) { + return default_value.str(); + } + + const char *result = ::mime_type_to_extension(mime_type.data(), mime_type.size()); + if (result != nullptr) { + return result; + } + + LOG(INFO) << "Unknown file MIME type " << mime_type; + return default_value.str(); +} + +string MimeType::from_extension(Slice extension, Slice default_value) { + if (extension.empty()) { + return default_value.str(); + } + + const char *result = ::extension_to_mime_type(extension.data(), extension.size()); + if (result != nullptr) { + return result; + } + + LOG(INFO) << "Unknown file extension " << extension; + return default_value.str(); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h new file mode 100644 index 0000000000..11210ceb30 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MimeType.h @@ -0,0 +1,20 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +class MimeType { + public: + static string to_extension(Slice mime_type, Slice default_value = Slice()); + static string from_extension(Slice extension, Slice default_value = Slice()); +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h new file mode 100644 index 0000000000..939bf51f28 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MovableValue.h @@ -0,0 +1,40 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +namespace td { + +template +class MovableValue { + public: + MovableValue() = default; + MovableValue(T val) : val_(val) { + } + MovableValue(MovableValue &&other) : val_(other.val_) { + other.clear(); + } + MovableValue &operator=(MovableValue &&other) { + val_ = other.val_; + other.clear(); + return *this; + } + MovableValue(const MovableValue &) = delete; + MovableValue &operator=(const MovableValue &) = delete; + ~MovableValue() = default; + + void clear() { + val_ = empty_val; + } + const T &get() const { + return val_; + } + + private: + T val_ = empty_val; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h new file mode 100644 index 0000000000..ae65554b72 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcQueue.h @@ -0,0 +1,449 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +// MPMC queue +// Simple semaphore protected implementation +// To close queue, one should send as much sentinel elements as there are readers. +// Once there are no readers and writers, one may easily destroy queue + +#include "td/utils/format.h" +#include "td/utils/HazardPointers.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" +#include "td/utils/ScopeGuard.h" + +#include +#include + +namespace td { + +namespace detail { +struct MpmcStat { + void alloc_ok(size_t thread_id) { + s(thread_id).alloc_ok_cnt++; + } + void alloc_error(size_t thread_id) { + s(thread_id).alloc_error_cnt++; + } + void push_loop_error(size_t thread_id) { + s(thread_id).push_loop_error_cnt++; + } + void push_loop_ok(size_t thread_id) { + s(thread_id).push_loop_ok_cnt++; + } + void dump() { + int alloc_ok_cnt = 0; + int alloc_error_cnt = 0; + int push_loop_error_cnt = 0; + int push_loop_ok_cnt = 0; + for (auto &d : arr) { + alloc_ok_cnt += d.alloc_ok_cnt; + alloc_error_cnt += d.alloc_error_cnt; + push_loop_error_cnt += d.push_loop_error_cnt; + push_loop_ok_cnt += d.push_loop_ok_cnt; + } + LOG(ERROR) << tag("alloc_ok_cnt", alloc_ok_cnt) << tag("alloc_error_cnt", alloc_error_cnt) + << tag("push_loop_error_cnt", push_loop_error_cnt) << tag("push_loop_ok_cnt", push_loop_ok_cnt); + } + + private: + struct ThreadStat { + int alloc_ok_cnt{0}; + int alloc_error_cnt{0}; + int push_loop_ok_cnt{0}; + int push_loop_error_cnt{0}; + char pad[TD_CONCURRENCY_PAD - sizeof(int) * 4]; + }; + std::array arr; + ThreadStat &s(size_t thread_id) { + return arr[thread_id]; + } +}; +} // namespace detail +//detail::MpmcStat stat_; + +template +class OneValue { + public: + bool set_value(T &value) { + value_ = std::move(value); + int state = Empty; + if (state_.compare_exchange_strong(state, Value, std::memory_order_acq_rel)) { + return true; + } + value = std::move(value_); + return false; + } + bool get_value(T &value) { + auto old_state = state_.exchange(Taken, std::memory_order_acq_rel); + if (old_state == Value) { + value = std::move(value_); + return true; + } + return false; + } + void reset() { + state_ = Empty; + value_ = T(); + } + + private: + enum Type : int { Empty = 0, Taken, Value }; + std::atomic state_{Empty}; + T value_; +}; + +template +class OneValue { + public: + bool set_value(T *value) { + T *was = nullptr; + return state_.compare_exchange_strong(was, value, std::memory_order_acq_rel); + } + bool get_value(T *&value) { + value = state_.exchange(Taken(), std::memory_order_acq_rel); + return value != nullptr; + } + void reset() { + state_ = nullptr; + } + OneValue() { + } + + private: + std::atomic state_{nullptr}; + T *Taken() { + static T xxx; + return &xxx; + } +}; + +template +class MpmcQueueBlock { + public: + explicit MpmcQueueBlock(size_t size) : nodes_(size) { + } + enum class PopStatus { Ok, Empty, Closed }; + + //blocking pop + //returns Ok or Closed + PopStatus pop(T &value) { + while (true) { + auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed); + if (read_pos >= nodes_.size()) { + return PopStatus::Closed; + } + //TODO blocking get_value + if (nodes_[static_cast(read_pos)].one_value.get_value(value)) { + return PopStatus::Ok; + } + } + } + + //nonblocking pop + //returns Ok, Empty or Closed + PopStatus try_pop(T &value) { + while (true) { + auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed); + if (read_pos >= nodes_.size()) { + return PopStatus::Closed; + } + if (nodes_[static_cast(read_pos)].one_value.get_value(value)) { + return PopStatus::Ok; + } + auto write_pos = write_pos_.load(std::memory_order_relaxed); + if (write_pos <= read_pos + 1) { + return PopStatus::Empty; + } + } + } + + enum class PushStatus { Ok, Closed }; + PushStatus push(T &value) { + while (true) { + auto write_pos = write_pos_.fetch_add(1, std::memory_order_relaxed); + if (write_pos >= nodes_.size()) { + return PushStatus::Closed; + } + if (nodes_[static_cast(write_pos)].one_value.set_value(value)) { + //stat_.push_loop_ok(0); + return PushStatus::Ok; + } + //stat_.push_loop_error(0); + } + } + + private: + struct Node { + OneValue one_value; + }; + std::atomic write_pos_{0}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + std::atomic read_pos_{0}; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + std::vector nodes_; + char pad3[TD_CONCURRENCY_PAD - sizeof(std::vector)]; +}; + +template +class MpmcQueueOld { + public: + explicit MpmcQueueOld(size_t threads_n) : MpmcQueueOld(1024, threads_n) { + } + static std::string get_description() { + return "Mpmc queue (fetch and add array queue)"; + } + MpmcQueueOld(size_t block_size, size_t threads_n) : block_size_{block_size}, hazard_pointers_{threads_n} { + auto node = std::make_unique(block_size_); + write_pos_ = node.get(); + read_pos_ = node.get(); + node.release(); + } + + MpmcQueueOld(const MpmcQueueOld &other) = delete; + MpmcQueueOld &operator=(const MpmcQueueOld &other) = delete; + MpmcQueueOld(MpmcQueueOld &&other) = delete; + MpmcQueueOld &operator=(MpmcQueueOld &&other) = delete; + ~MpmcQueueOld() { + auto *ptr = read_pos_.load(std::memory_order_relaxed); + while (ptr) { + auto *to_delete = ptr; + ptr = ptr->next_.load(std::memory_order_relaxed); + delete to_delete; + } + //stat_.dump(); + //stat_ = MpmcStat(); + } + + size_t hazard_pointers_to_delele_size_unsafe() const { + return hazard_pointers_.to_delete_size_unsafe(); + } + void gc(size_t thread_id) { + hazard_pointers_.retire(thread_id); + } + + using PushStatus = typename MpmcQueueBlock::PushStatus; + using PopStatus = typename MpmcQueueBlock::PopStatus; + + void push(T value, size_t thread_id) { + auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0); + while (true) { + auto node = hazard_ptr_holder.protect(write_pos_); + auto status = node->block.push(value); + switch (status) { + case PushStatus::Ok: + return; + case PushStatus::Closed: { + auto next = node->next_.load(std::memory_order_acquire); + if (next == nullptr) { + auto new_node = new Node(block_size_); + new_node->block.push(value); + if (node->next_.compare_exchange_strong(next, new_node, std::memory_order_acq_rel)) { + //stat_.alloc_ok(thread_id); + write_pos_.compare_exchange_strong(node, new_node, std::memory_order_acq_rel); + return; + } else { + //stat_.alloc_error(thread_id); + new_node->block.pop(value); + //CHECK(status == PopStatus::Ok); + delete new_node; + } + } + //CHECK(next != nullptr); + write_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel); + break; + } + } + } + } + + bool try_pop(T &value, size_t thread_id) { + auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0); + while (true) { + auto node = hazard_ptr_holder.protect(read_pos_); + auto status = node->block.try_pop(value); + switch (status) { + case PopStatus::Ok: + return true; + case PopStatus::Empty: + return false; + case PopStatus::Closed: { + auto next = node->next_.load(std::memory_order_acquire); + if (!next) { + return false; + } + if (read_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel)) { + hazard_ptr_holder.clear(); + hazard_pointers_.retire(thread_id, node); + } + break; + } + } + } + } + + T pop(size_t thread_id) { + T value; + while (true) { + if (try_pop(value, thread_id)) { + return value; + } + td::this_thread::yield(); + } + } + + private: + struct Node { + explicit Node(size_t block_size) : block{block_size} { + } + std::atomic next_{nullptr}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + MpmcQueueBlock block; + //Got pad in MpmcQueueBlock + }; + std::atomic write_pos_; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + std::atomic read_pos_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + size_t block_size_; + HazardPointers hazard_pointers_; + //Got pad in HazardPointers +}; + +template +class MpmcQueue { + public: + explicit MpmcQueue(size_t threads_n) : MpmcQueue(1024, threads_n) { + } + static std::string get_description() { + return "NEW Mpmc queue (fetch and add array queue)"; + } + MpmcQueue(size_t block_size, size_t threads_n) : hazard_pointers_{threads_n} { + auto node = std::make_unique(); + write_pos_ = node.get(); + read_pos_ = node.get(); + node.release(); + } + + MpmcQueue(const MpmcQueue &other) = delete; + MpmcQueue &operator=(const MpmcQueue &other) = delete; + MpmcQueue(MpmcQueue &&other) = delete; + MpmcQueue &operator=(MpmcQueue &&other) = delete; + ~MpmcQueue() { + auto *ptr = read_pos_.load(std::memory_order_relaxed); + while (ptr) { + auto *to_delete = ptr; + ptr = ptr->next.load(std::memory_order_relaxed); + delete to_delete; + } + } + + size_t hazard_pointers_to_delele_size_unsafe() const { + return hazard_pointers_.to_delete_size_unsafe(); + } + void gc(size_t thread_id) { + hazard_pointers_.retire(thread_id); + } + + void push(T value, size_t thread_id) { + SCOPE_EXIT { + hazard_pointers_.clear(thread_id, 0); + }; + while (true) { + auto node = hazard_pointers_.protect(thread_id, 0, write_pos_); + auto &block = node->block; + auto pos = block.write_pos++; + if (pos >= block.data.size()) { + auto next = node->next.load(); + if (next == nullptr) { + auto new_node = new Node{}; + new_node->block.write_pos++; + new_node->block.data[0].set_value(value); + Node *null = nullptr; + if (node->next.compare_exchange_strong(null, new_node)) { + write_pos_.compare_exchange_strong(node, new_node); + return; + } else { + new_node->block.data[0].get_value(value); + delete new_node; + } + } else { + write_pos_.compare_exchange_strong(node, next); + } + } else { + if (block.data[static_cast(pos)].set_value(value)) { + return; + } + } + } + } + + bool try_pop(T &value, size_t thread_id) { + SCOPE_EXIT { + hazard_pointers_.clear(thread_id, 0); + }; + while (true) { + auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); + auto &block = node->block; + if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + return false; + } + auto pos = block.read_pos++; + if (pos >= block.data.size()) { + auto next = node->next.load(); + if (!next) { + return false; + } + if (read_pos_.compare_exchange_strong(node, next)) { + hazard_pointers_.clear(thread_id, 0); + hazard_pointers_.retire(thread_id, node); + } + } else { + if (block.data[static_cast(pos)].get_value(value)) { + return true; + } + } + } + } + + T pop(size_t thread_id) { + T value; + while (true) { + if (try_pop(value, thread_id)) { + return value; + } + td::this_thread::yield(); + } + } + + private: + struct Block { + std::atomic write_pos{0}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + std::atomic read_pos{0}; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + std::array, 1024> data; + char pad3[TD_CONCURRENCY_PAD]; + }; + struct Node { + Node() = default; + + Block block; + std::atomic next{nullptr}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + //Got pad in MpmcQueueBlock + }; + std::atomic write_pos_; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + std::atomic read_pos_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic)]; + HazardPointers hazard_pointers_; + //Got pad in HazardPointers +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h new file mode 100644 index 0000000000..0f48620e63 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpmcWaiter.h @@ -0,0 +1,106 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/thread.h" + +#include +#include +#include + +namespace td { + +class MpmcWaiter { + public: + int wait(int yields, uint32 worker_id) { + if (yields < RoundsTillSleepy) { + td::this_thread::yield(); + return yields + 1; + } else if (yields == RoundsTillSleepy) { + auto state = state_.load(std::memory_order_relaxed); + if (!State::has_worker(state)) { + auto new_state = State::with_worker(state, worker_id); + if (state_.compare_exchange_strong(state, new_state)) { + td::this_thread::yield(); + return yields + 1; + } + if (state == State::awake()) { + return 0; + } + } + td::this_thread::yield(); + return 0; + } else if (yields < RoundsTillAsleep) { + auto state = state_.load(std::memory_order_acquire); + if (State::still_sleepy(state, worker_id)) { + td::this_thread::yield(); + return yields + 1; + } + return 0; + } else { + auto state = state_.load(std::memory_order_acquire); + if (State::still_sleepy(state, worker_id)) { + std::unique_lock lock(mutex_); + if (state_.compare_exchange_strong(state, State::asleep(), std::memory_order_acq_rel)) { + condition_variable_.wait(lock); + } + } + return 0; + } + } + + int stop_wait(int yields, uint32 worker_id) { + if (yields > RoundsTillSleepy) { + notify_cold(); + } + return 0; + } + + void notify() { + if (state_.load(std::memory_order_acquire) == State::awake()) { + return; + } + notify_cold(); + } + + private: + struct State { + static constexpr uint32 awake() { + return 0; + } + static constexpr uint32 asleep() { + return 1; + } + static bool is_asleep(uint32 state) { + return (state & 1) != 0; + } + static bool has_worker(uint32 state) { + return (state >> 1) != 0; + } + static int32 with_worker(uint32 state, uint32 worker) { + return state | ((worker + 1) << 1); + } + static bool still_sleepy(uint32 state, uint32 worker) { + return (state >> 1) == (worker + 1); + } + }; + enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 }; + std::atomic state_{State::awake()}; + std::mutex mutex_; + std::condition_variable condition_variable_; + + void notify_cold() { + auto old_state = state_.exchange(State::awake(), std::memory_order_release); + if (State::is_asleep(old_state)) { + std::lock_guard guard(mutex_); + condition_variable_.notify_all(); + } + } +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h new file mode 100644 index 0000000000..4398c7503d --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscLinkQueue.h @@ -0,0 +1,174 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include + +namespace td { +//NB: holder of the queue holds all responsibility of freeing its nodes +class MpscLinkQueueImpl { + public: + class Node; + class Reader; + + void push(Node *node) { + node->next_ = head_.load(std::memory_order_relaxed); + while (!head_.compare_exchange_strong(node->next_, node, std::memory_order_release, std::memory_order_relaxed)) { + } + } + + void push_unsafe(Node *node) { + node->next_ = head_.load(std::memory_order_relaxed); + head_.store(node, std::memory_order_relaxed); + } + + void pop_all(Reader &reader) { + return reader.add(head_.exchange(nullptr, std::memory_order_acquire)); + } + + void pop_all_unsafe(Reader &reader) { + return reader.add(head_.exchange(nullptr, std::memory_order_relaxed)); + } + + class Node { + friend class MpscLinkQueueImpl; + Node *next_{nullptr}; + }; + + class Reader { + public: + Node *read() { + auto old_head = head_; + if (head_) { + head_ = head_->next_; + } + return old_head; + } + void delay(Node *node) { + node->next_ = head_; + if (!head_) { + tail_ = node; + } + head_ = node; + } + size_t calc_size() const { + size_t res = 0; + for (auto it = head_; it != nullptr; it = it->next_, res++) { + } + return res; + } + + private: + friend class MpscLinkQueueImpl; + void add(Node *node) { + if (node == nullptr) { + return; + } + // Reverse list + Node *tail = node; + Node *head = nullptr; + while (node) { + auto next = node->next_; + node->next_ = head; + head = node; + node = next; + } + if (head_ == nullptr) { + head_ = head; + } else { + tail_->next_ = head; + } + tail_ = tail; + } + Node *head_{nullptr}; + Node *tail_{nullptr}; + }; + + private: + std::atomic head_{nullptr}; +}; + +// Uses MpscLinkQueueImpl. +// Node should have to_mpsc_link_queue_node and from_mpsc_link_queue_node functions +template +class MpscLinkQueue { + public: + void push(Node node) { + impl_.push(node.to_mpsc_link_queue_node()); + } + void push_unsafe(Node node) { + impl_.push_unsafe(node.to_mpsc_link_queue_node()); + } + class Reader { + public: + ~Reader() { + CHECK(!read()); + } + Node read() { + auto node = impl_.read(); + if (!node) { + return {}; + } + return Node::from_mpsc_link_queue_node(node); + } + void delay(Node node) { + impl_.delay(node.to_mpsc_link_queue_node()); + } + size_t calc_size() const { + return impl_.calc_size(); + } + + private: + friend class MpscLinkQueue; + + MpscLinkQueueImpl::Reader impl_; + MpscLinkQueueImpl::Reader &impl() { + return impl_; + } + }; + + void pop_all(Reader &reader) { + return impl_.pop_all(reader.impl()); + } + void pop_all_unsafe(Reader &reader) { + return impl_.pop_all_unsafe(reader.impl()); + } + + private: + MpscLinkQueueImpl impl_; +}; + +template +class MpscLinkQueueUniquePtrNode { + public: + MpscLinkQueueUniquePtrNode() = default; + explicit MpscLinkQueueUniquePtrNode(std::unique_ptr ptr) : ptr_(std::move(ptr)) { + } + + MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() { + return ptr_.release()->to_mpsc_link_queue_node(); + } + static MpscLinkQueueUniquePtrNode from_mpsc_link_queue_node(td::MpscLinkQueueImpl::Node *node) { + return MpscLinkQueueUniquePtrNode(std::unique_ptr(Value::from_mpsc_link_queue_node(node))); + } + + explicit operator bool() { + return ptr_ != nullptr; + } + + Value &value() { + return *ptr_; + } + + private: + std::unique_ptr ptr_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h new file mode 100644 index 0000000000..89d2df8693 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/MpscPollableQueue.h @@ -0,0 +1,154 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/misc.h" +#include "td/utils/port/EventFd.h" +#include "td/utils/SpinLock.h" + +#if !TD_EVENTFD_UNSUPPORTED +#if !TD_WINDOWS +#include +#include +#endif + +#include + +namespace td { +// interface like in PollableQueue +template +class MpscPollableQueue { + public: + int reader_wait_nonblock() { + auto ready = reader_vector_.size() - reader_pos_; + if (ready != 0) { + return narrow_cast(ready); + } + + auto guard = lock_.lock(); + if (writer_vector_.empty()) { + event_fd_.acquire(); + wait_event_fd_ = true; + return 0; + } else { + reader_vector_.clear(); + reader_pos_ = 0; + std::swap(writer_vector_, reader_vector_); + return narrow_cast(reader_vector_.size()); + } + } + ValueT reader_get_unsafe() { + return std::move(reader_vector_[reader_pos_++]); + } + void reader_flush() { + //nop + } + void writer_put(ValueT value) { + auto guard = lock_.lock(); + writer_vector_.push_back(std::move(value)); + if (wait_event_fd_) { + wait_event_fd_ = false; + event_fd_.release(); + } + } + EventFd &reader_get_event_fd() { + return event_fd_; + } + void writer_flush() { + //nop + } + + void init() { + event_fd_.init(); + } + void destroy() { + if (!event_fd_.empty()) { + event_fd_.close(); + wait_event_fd_ = false; + writer_vector_.clear(); + reader_vector_.clear(); + reader_pos_ = 0; + } + } + +// Just example of usage +#if !TD_WINDOWS + int reader_wait() { + int res; + + while ((res = reader_wait_nonblock()) == 0) { + // TODO: reader_flush? + pollfd fd; + fd.fd = reader_get_event_fd().get_fd().get_native_fd(); + fd.events = POLLIN; + poll(&fd, 1, -1); + } + return res; + } +#endif + + private: + SpinLock lock_; + bool wait_event_fd_{false}; + EventFd event_fd_; + std::vector writer_vector_; + std::vector reader_vector_; + size_t reader_pos_{0}; +}; + +} // namespace td + +#else +#include "td/utils/logging.h" + +namespace td { + +// dummy implementation which shouldn't be used + +template +class MpscPollableQueue { + public: + using ValueType = T; + + void init() { + UNREACHABLE(); + } + + template + void writer_put(PutValueType &&value) { + UNREACHABLE(); + } + + void writer_flush() { + UNREACHABLE(); + } + + int reader_wait_nonblock() { + UNREACHABLE(); + return 0; + } + + ValueType reader_get_unsafe() { + UNREACHABLE(); + return ValueType(); + } + + void reader_flush() { + UNREACHABLE(); + } + + MpscPollableQueue() = default; + MpscPollableQueue(const MpscPollableQueue &) = delete; + MpscPollableQueue &operator=(const MpscPollableQueue &) = delete; + MpscPollableQueue(MpscPollableQueue &&) = delete; + MpscPollableQueue &operator=(MpscPollableQueue &&) = delete; + ~MpscPollableQueue() = default; +}; + +} // namespace td + +#endif diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h new file mode 100644 index 0000000000..202de5f7d4 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Named.h @@ -0,0 +1,27 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +class Named { + public: + Slice get_name() const { + return name_; + } + void set_name(Slice name) { + name_ = name.str(); + } + + private: + string name_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h new file mode 100644 index 0000000000..e6e4549dbb --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/ObjectPool.h @@ -0,0 +1,249 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include +#include +#include + +namespace td { +// It is draft object pool implementaion +// +// Compared with std::shared_ptr: +// + WeakPtr are much faster. Just pointer copy. No barriers, no atomics. +// - We can't destroy object, because we don't know if it is pointed to by some weak pointer +// +template +class ObjectPool { + struct Storage; + + public: + class WeakPtr { + public: + WeakPtr() : generation_(-1), storage_(nullptr) { + } + WeakPtr(int32 generation, Storage *storage) : generation_(generation), storage_(storage) { + } + + DataT &operator*() const { + return storage_->data; + } + + DataT *operator->() const { + return &**this; + } + + // Pattern of usage: 1. Read an object 2. Check if read was valid via is_alive + // + // It is not very usual case of acquire/release use. + // Instead of publishing an object via some flag we do the opposite. + // We publish new generation via destruction of the data. + // In usual case if we see a flag then we are able to use an object. + // In our case if we have used an object and it is already invalid, then generation will mismatch + bool is_alive() const { + if (!storage_) { + return false; + } + std::atomic_thread_fence(std::memory_order_acquire); + return generation_ == storage_->generation.load(std::memory_order_relaxed); + } + + // Used for ActorId + bool is_alive_unsafe() const { + if (!storage_) { + return false; + } + return generation_ == storage_->generation.load(std::memory_order_relaxed); + } + + bool empty() const { + return storage_ == nullptr; + } + void clear() { + generation_ = -1; + storage_ = nullptr; + } + int32 generation() { + return generation_; + } + + private: + int32 generation_; + Storage *storage_; + }; + + class OwnerPtr { + public: + OwnerPtr() = default; + OwnerPtr(const OwnerPtr &) = delete; + OwnerPtr &operator=(const OwnerPtr &) = delete; + OwnerPtr(OwnerPtr &&other) : storage_(other.storage_), parent_(other.parent_) { + other.storage_ = nullptr; + other.parent_ = nullptr; + } + OwnerPtr &operator=(OwnerPtr &&other) { + if (this != &other) { + storage_ = other.storage_; + parent_ = other.parent_; + other.storage_ = nullptr; + other.parent_ = nullptr; + } + return *this; + } + ~OwnerPtr() { + reset(); + } + + DataT *get() { + return &storage_->data; + } + DataT &operator*() { + return *get(); + } + DataT *operator->() { + return get(); + } + + const DataT *get() const { + return &storage_->data; + } + const DataT &operator*() const { + return *get(); + } + const DataT *operator->() const { + return get(); + } + + WeakPtr get_weak() { + return WeakPtr(storage_->generation.load(std::memory_order_relaxed), storage_); + } + int32 generation() { + return storage_->generation.load(std::memory_order_relaxed); + } + + Storage *release() { + auto result = storage_; + storage_ = nullptr; + return result; + } + + bool empty() const { + return storage_ == nullptr; + } + + void reset() { + if (storage_ != nullptr) { + // for crazy cases when data owns owner pointer to itself. + auto tmp = storage_; + storage_ = nullptr; + parent_->release(OwnerPtr(tmp, parent_)); + } + } + + private: + friend class ObjectPool; + OwnerPtr(Storage *storage, ObjectPool *parent) : storage_(storage), parent_(parent) { + } + Storage *storage_ = nullptr; + ObjectPool *parent_ = nullptr; + }; + + template + OwnerPtr create(ArgsT &&... args) { + Storage *storage = get_storage(); + storage->init_data(std::forward(args)...); + return OwnerPtr(storage, this); + } + + OwnerPtr create_empty() { + Storage *storage = get_storage(); + return OwnerPtr(storage, this); + } + + void set_check_empty(bool flag) { + check_empty_flag_ = flag; + } + + void release(OwnerPtr &&owner_ptr) { + Storage *storage = owner_ptr.release(); + storage->destroy_data(); + release_storage(storage); + } + + ObjectPool() = default; + ObjectPool(const ObjectPool &) = delete; + ObjectPool &operator=(const ObjectPool &) = delete; + ObjectPool(ObjectPool &&other) = delete; + ObjectPool &operator=(ObjectPool &&other) = delete; + ~ObjectPool() { + while (head_.load()) { + auto to_delete = head_.load(); + head_ = to_delete->next; + delete to_delete; + storage_count_--; + } + CHECK(storage_count_.load() == 0) << storage_count_.load(); + } + + private: + struct Storage { + // union { + DataT data; + //}; + Storage *next = nullptr; + std::atomic generation{1}; + + template + void init_data(ArgsT &&... args) { + // new (&data) DataT(std::forward(args)...); + data = DataT(std::forward(args)...); + } + void destroy_data() { + generation.fetch_add(1, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_release); + data.clear(); + } + }; + + std::atomic storage_count_{0}; + std::atomic head_{static_cast(nullptr)}; + bool check_empty_flag_ = false; + + // TODO(perf): allocation Storages in chunks? Anyway we won't be able to release them. + // TODO(perf): memory order + // TODO(perf): use another non lockfree list for release on the same thread + // only one thread, so no aba problem + Storage *get_storage() { + if (head_.load() == nullptr) { + storage_count_++; + return new Storage(); + } + Storage *res; + while (true) { + res = head_.load(); + auto *next = res->next; + if (head_.compare_exchange_weak(res, next)) { + break; + } + } + return res; + } + // release can be called from other thread + void release_storage(Storage *storage) { + while (true) { + auto *save_head = head_.load(); + storage->next = save_head; + if (head_.compare_exchange_weak(save_head, storage)) { + break; + } + } + } +}; +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h new file mode 100644 index 0000000000..8511e0ce8b --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/Observer.h @@ -0,0 +1,41 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +namespace td { + +class ObserverBase { + public: + ObserverBase() = default; + ObserverBase(const ObserverBase &) = delete; + ObserverBase &operator=(const ObserverBase &) = delete; + ObserverBase(ObserverBase &&) = delete; + ObserverBase &operator=(ObserverBase &&) = delete; + virtual ~ObserverBase() = default; + + virtual void notify() = 0; +}; + +class Observer : ObserverBase { + public: + Observer() = default; + explicit Observer(unique_ptr &&ptr) : observer_ptr_(std::move(ptr)) { + } + + void notify() override { + if (observer_ptr_) { + observer_ptr_->notify(); + } + } + + private: + unique_ptr observer_ptr_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionsParser.h b/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionsParser.h new file mode 100644 index 0000000000..6ac0385575 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdutils/td/utils/OptionsParser.h @@ -0,0 +1,150 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include +#include + +#if !TD_WINDOWS +#include +#endif + +namespace td { + +class OptionsParser { + public: + class Option { + public: + enum Type { NoArg, Arg, OptionalArg }; + Type type; + char short_key; + std::string long_key; + std::string description; + std::function arg_callback; + }; + + void set_description(std::string description) { + description_ = std::move(description); + } + + void add_option(Option::Type type, char short_key, Slice long_key, Slice description, + std::function callback) { + options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)}); + } + + void add_option(char short_key, Slice long_key, Slice description, std::function callback) { + add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback)); + } + + void add_option(char short_key, Slice long_key, Slice description, std::function callback) { + // Ouch. There must be some better way + add_option(Option::Type::NoArg, short_key, long_key, description, + std::bind([](std::function &func, Slice) { return func(); }, std::move(callback), + std::placeholders::_1)); + } + + Result run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT { +#if TD_WINDOWS + return -1; +#else + // use getopt. long keys are not supported for now + char buff[1024]; + StringBuilder sb({buff, sizeof(buff)}); + for (auto &opt : options_) { + CHECK(opt.type != Option::OptionalArg); + sb << opt.short_key; + if (opt.type == Option::Arg) { + sb << ":"; + } + } + if (sb.is_error()) { + return Status::Error("Can't parse options"); + } + CSlice short_options = sb.as_cslice(); + + vector