diff options
author | Mataes <mataes2007@gmail.com> | 2018-04-27 20:39:22 +0300 |
---|---|---|
committer | Mataes <mataes2007@gmail.com> | 2018-04-27 20:39:22 +0300 |
commit | b9ce1d4d98525490ca1a38e2d9fd4f3369adb3e0 (patch) | |
tree | 787c80a909776c1c4d099b638c83c7977bb070e2 /libs/tdlib/td/tdutils | |
parent | 5ed0126c16d061d6e87aa20c718e14608c66feec (diff) |
added tdlib library
Diffstat (limited to 'libs/tdlib/td/tdutils')
171 files changed, 24215 insertions, 0 deletions
diff --git a/libs/tdlib/td/tdutils/CMakeLists.txt b/libs/tdlib/td/tdutils/CMakeLists.txt new file mode 100644 index 0000000000..1fbc34df32 --- /dev/null +++ b/libs/tdlib/td/tdutils/CMakeLists.txt @@ -0,0 +1,244 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +if (NOT ZLIB_FOUND) + find_package(ZLIB) +endif() +if (ZLIB_FOUND) + set(TD_HAVE_ZLIB 1) + message(STATUS "Found ZLIB: ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES}") + + # OpenSSL internally depends on zlib + if (NOT OPENSSL_FOUND) + find_package(OpenSSL) + endif() + if (OPENSSL_FOUND) + set(TD_HAVE_OPENSSL 1) + endif() +endif() + +configure_file(td/utils/config.h.in td/utils/config.h @ONLY) + +add_subdirectory(generate) + +# TDUTILS +set_source_files_properties(${TDMIME_AUTO} PROPERTIES GENERATED TRUE) +if (CLANG OR GCC) + set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-conversion") +elseif (MSVC) + set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " /wd4267") +endif() +if (CLANG) + set_property(SOURCE ${TDMIME_AUTO} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-deprecated-register") +endif() + +set(TDUTILS_SOURCE + td/utils/port/Clocks.cpp + td/utils/port/Fd.cpp + td/utils/port/FileFd.cpp + td/utils/port/IPAddress.cpp + td/utils/port/path.cpp + td/utils/port/ServerSocketFd.cpp + td/utils/port/signals.cpp + td/utils/port/sleep.cpp + td/utils/port/SocketFd.cpp + td/utils/port/Stat.cpp + td/utils/port/thread_local.cpp + td/utils/port/wstring_convert.cpp + + td/utils/port/detail/Epoll.cpp + td/utils/port/detail/EventFdBsd.cpp + td/utils/port/detail/EventFdLinux.cpp + td/utils/port/detail/EventFdWindows.cpp + td/utils/port/detail/KQueue.cpp + td/utils/port/detail/Poll.cpp + td/utils/port/detail/Select.cpp + td/utils/port/detail/ThreadIdGuard.cpp + td/utils/port/detail/WineventPoll.cpp + + ${TDMIME_AUTO} + + td/utils/base64.cpp + td/utils/BigNum.cpp + td/utils/buffer.cpp + td/utils/crypto.cpp + td/utils/FileLog.cpp + td/utils/filesystem.cpp + td/utils/find_boundary.cpp + td/utils/Gzip.cpp + td/utils/GzipByteFlow.cpp + td/utils/Hints.cpp + td/utils/HttpUrl.cpp + td/utils/JsonBuilder.cpp + td/utils/logging.cpp + td/utils/misc.cpp + td/utils/MimeType.cpp + td/utils/Random.cpp + td/utils/StackAllocator.cpp + td/utils/Status.cpp + td/utils/StringBuilder.cpp + td/utils/Time.cpp + td/utils/Timer.cpp + td/utils/tl_parsers.cpp + td/utils/unicode.cpp + td/utils/utf8.cpp + + td/utils/port/Clocks.h + td/utils/port/config.h + td/utils/port/CxCli.h + td/utils/port/EventFd.h + td/utils/port/EventFdBase.h + td/utils/port/Fd.h + td/utils/port/FileFd.h + td/utils/port/IPAddress.h + td/utils/port/path.h + td/utils/port/platform.h + td/utils/port/Poll.h + td/utils/port/PollBase.h + td/utils/port/RwMutex.h + td/utils/port/ServerSocketFd.h + td/utils/port/signals.h + td/utils/port/sleep.h + td/utils/port/SocketFd.h + td/utils/port/Stat.h + td/utils/port/thread.h + td/utils/port/thread_local.h + td/utils/port/wstring_convert.h + + td/utils/port/detail/Epoll.h + td/utils/port/detail/EventFdBsd.h + td/utils/port/detail/EventFdLinux.h + td/utils/port/detail/EventFdWindows.h + td/utils/port/detail/KQueue.h + td/utils/port/detail/Poll.h + td/utils/port/detail/Select.h + td/utils/port/detail/ThreadIdGuard.h + td/utils/port/detail/ThreadPthread.h + td/utils/port/detail/ThreadStl.h + td/utils/port/detail/WineventPoll.h + + td/utils/AesCtrByteFlow.h + td/utils/base64.h + td/utils/benchmark.h + td/utils/BigNum.h + td/utils/buffer.h + td/utils/BufferedFd.h + td/utils/BufferedReader.h + td/utils/ByteFlow.h + td/utils/ChangesProcessor.h + td/utils/Closure.h + td/utils/common.h + td/utils/Container.h + td/utils/crypto.h + td/utils/Enumerator.h + td/utils/FileLog.h + td/utils/filesystem.h + td/utils/find_boundary.h + td/utils/FloodControlFast.h + td/utils/FloodControlStrict.h + td/utils/format.h + td/utils/Gzip.h + td/utils/GzipByteFlow.h + td/utils/HazardPointers.h + td/utils/Heap.h + td/utils/Hints.h + td/utils/HttpUrl.h + td/utils/int_types.h + td/utils/invoke.h + td/utils/JsonBuilder.h + td/utils/List.h + td/utils/logging.h + td/utils/MemoryLog.h + td/utils/MimeType.h + td/utils/misc.h + td/utils/MovableValue.h + td/utils/MpmcQueue.h + td/utils/MpmcWaiter.h + td/utils/MpscPollableQueue.h + td/utils/MpscLinkQueue.h + td/utils/Named.h + td/utils/ObjectPool.h + td/utils/Observer.h + td/utils/optional.h + td/utils/OptionsParser.h + td/utils/OrderedEventsProcessor.h + td/utils/overloaded.h + td/utils/Parser.h + td/utils/PathView.h + td/utils/queue.h + td/utils/Random.h + td/utils/ScopeGuard.h + td/utils/SharedObjectPool.h + td/utils/Slice-decl.h + td/utils/Slice.h + td/utils/SpinLock.h + td/utils/StackAllocator.h + td/utils/Status.h + td/utils/Storer.h + td/utils/StorerBase.h + td/utils/StringBuilder.h + td/utils/tests.h + td/utils/Time.h + td/utils/TimedStat.h + td/utils/Timer.h + td/utils/tl_helpers.h + td/utils/tl_parsers.h + td/utils/tl_storers.h + td/utils/type_traits.h + td/utils/unicode.h + td/utils/utf8.h + td/utils/Variant.h +) + +set(TDUTILS_TEST_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/test/crypto.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/Enumerator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/filesystem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/gzip.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/HazardPointers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/heap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/json.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/misc.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/SharedObjectPool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test/variant.cpp + PARENT_SCOPE +) + +#RULES +#LIBRARIES +add_library(tdutils STATIC ${TDUTILS_SOURCE}) +if (WIN32) + # find_library(WS2_32_LIBRARY ws2_32) + # find_library(MSWSOCK_LIBRARY Mswsock) + # target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY}) + target_link_libraries(tdutils PRIVATE ws2_32 Mswsock) +endif() +if (NOT CMAKE_CROSSCOMPILING) + add_dependencies(tdutils tdmime_auto) +endif() + +if (DEFINED CMAKE_THREAD_LIBS_INIT) + target_link_libraries(tdutils PUBLIC ${CMAKE_THREAD_LIBS_INIT}) +endif() +target_include_directories(tdutils PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>) + +if (OPENSSL_FOUND) + target_link_libraries(tdutils PRIVATE ${OPENSSL_CRYPTO_LIBRARY} ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) + target_include_directories(tdutils SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) +endif() + +if (ZLIB_FOUND) + target_link_libraries(tdutils PRIVATE ${ZLIB_LIBRARIES}) + target_include_directories(tdutils SYSTEM PRIVATE ${ZLIB_INCLUDE_DIR}) +endif() + +install(TARGETS tdutils EXPORT TdTargets + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + RUNTIME DESTINATION bin + INCLUDES DESTINATION include +) diff --git a/libs/tdlib/td/tdutils/generate/CMakeLists.txt b/libs/tdlib/td/tdutils/generate/CMakeLists.txt new file mode 100644 index 0000000000..69697b04b8 --- /dev/null +++ b/libs/tdlib/td/tdutils/generate/CMakeLists.txt @@ -0,0 +1,64 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +# Generates files for MIME type <-> extension conversions +# DEPENDS ON: gperf grep bash/powershell + +file(MAKE_DIRECTORY auto) + +set(TDMIME_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp +) +set(TDMIME_AUTO + ${TDMIME_SOURCE} + PARENT_SCOPE +) + +add_custom_target(tdmime_auto DEPENDS ${TDMIME_SOURCE}) + +if (NOT CMAKE_CROSSCOMPILING) + find_program(GPERF_EXECUTABLE gperf) + if (NOT GPERF_EXECUTABLE) + message(FATAL_ERROR "Could NOT find gperf. Path to gperf needs to be specified manually, i.e. 'cmake -DGPERF_EXECUTABLE:FILEPATH=\"<path to gperf executable>\" .')") + endif() + + set(GPERF_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.gperf + ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.gperf + ) + + set(GPERF_GEN_SOURCE generate_mime_types_gperf.cpp) + + add_executable(generate_mime_types_gperf ${GPERF_GEN_SOURCE}) + + add_custom_command( + OUTPUT ${GPERF_FILES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND generate_mime_types_gperf mime_types.txt ${GPERF_FILES} + DEPENDS generate_mime_types_gperf mime_types.txt + ) + + if (WIN32) + set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/mime_type_to_extension.cpp auto/mime_type_to_extension.gperf) + else() + set(MIME_TYPE_TO_EXTENSION_CMD ${GPERF_EXECUTABLE} -m100 auto/mime_type_to_extension.gperf | grep -v __gnu_inline__ > auto/mime_type_to_extension.cpp) + endif() + add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/mime_type_to_extension.cpp + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${MIME_TYPE_TO_EXTENSION_CMD} + DEPENDS auto/mime_type_to_extension.gperf + ) + + if (WIN32) + set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 --output-file=auto/extension_to_mime_type.cpp auto/extension_to_mime_type.gperf) + else() + set(EXTENSION_TO_MIME_TYPE_CMD ${GPERF_EXECUTABLE} -m100 auto/extension_to_mime_type.gperf | grep -v __gnu_inline__ > auto/extension_to_mime_type.cpp) + endif() + add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/auto/extension_to_mime_type.cpp + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND ${EXTENSION_TO_MIME_TYPE_CMD} + DEPENDS auto/extension_to_mime_type.gperf + ) +endif() diff --git a/libs/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp b/libs/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp new file mode 100644 index 0000000000..ac7ff68605 --- /dev/null +++ b/libs/tdlib/td/tdutils/generate/generate_mime_types_gperf.cpp @@ -0,0 +1,146 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <fstream> +#include <iostream> +#include <map> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +std::pair<std::string, std::string> split(std::string s, char delimiter = ' ') { + auto delimiter_pos = s.find(delimiter); + if (delimiter_pos == std::string::npos) { + return {std::move(s), ""}; + } else { + auto head = s.substr(0, delimiter_pos); + auto tail = s.substr(delimiter_pos + 1); + return {head, tail}; + } +} + +bool generate(const char *file_name, const char *from_name, const char *to_name, + const std::map<std::string, std::string> &map) { + // binary mode is needed for MSYS2 gperf + std::ofstream out(file_name, std::ios_base::trunc | std::ios_base::binary); + if (!out) { + std::cerr << "Can't open output file \"" << file_name << std::endl; + return false; + } + + out << "%struct-type\n"; + out << "%ignore-case\n"; + out << "%language=ANSI-C\n"; + out << "%readonly-tables\n"; + out << "%includes\n"; + out << "%enum\n"; + out << "%define slot-name " << from_name << "\n"; + out << "%define initializer-suffix ,nullptr\n"; + out << "%define slot-name " << from_name << "\n"; + out << "%define hash-function-name " << from_name << "_hash\n"; + out << "%define lookup-function-name search_" << from_name << "\n"; + // out << "%define class-name " << from_name << "_to_" << to_name << "\n"; + out << "struct " << from_name << "_and_" << to_name << " {\n"; + out << " const char *" << from_name << ";\n"; + out << " const char *" << to_name << ";\n"; + out << "}\n"; + out << "%%\n"; + + for (auto &value : map) { + out << '"' << value.first << "\", \"" << value.second << '"' << "\n"; + } + + out << "%%\n"; + out << "const char *" << from_name << "_to_" << to_name << "(const char *" << from_name << ", size_t " << from_name + << "_len) {\n"; + out << " const auto &result = search_" << from_name << "(" << from_name << ", " << from_name << "_len);\n"; + out << " if (result == nullptr) {\n"; + out << " return nullptr;\n"; + out << " }\n"; + out << "\n"; + out << " return result->" << to_name << ";\n"; + out << "}\n"; + + return true; +} + +int main(int argc, char *argv[]) { + if (argc != 4) { + std::cerr << "Wrong number of arguments supplied. Expected 'generate_mime_types_gperf <mime_types.txt> " + "<mime_type_to_extension.cpp> <extension_to_mime_type.cpp>'" + << std::endl; + return EXIT_FAILURE; + } + + std::ifstream mime_types_file(argv[1]); + if (!mime_types_file) { + std::cerr << "Can't open input file \"" << argv[1] << std::endl; + return EXIT_FAILURE; + } + + std::map<std::string, std::string> mime_type_to_extension; + std::map<std::string, std::string> extension_to_mime_type; + + std::string line; + while (std::getline(mime_types_file, line)) { + while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) { + line.pop_back(); + } + + std::string mime_type; + std::string extensions_string; + std::tie(mime_type, extensions_string) = split(line, '\t'); + + if (mime_type.empty()) { + std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl; + continue; + } + + auto extensions_start_position = extensions_string.find_first_not_of(" \t"); + if (extensions_start_position == std::string::npos) { + std::cerr << "Wrong MIME type description \"" << line << "\"" << std::endl; + continue; + } + extensions_string = extensions_string.substr(extensions_start_position); + + std::vector<std::string> extensions; + while (!extensions_string.empty()) { + extensions.push_back(""); + std::tie(extensions.back(), extensions_string) = split(extensions_string); + } + assert(!extensions.empty()); + + std::map<std::string, std::string> preffered_extensions{{"image/jpeg", "jpg"}, {"audio/mpeg", "mp3"}, + {"audio/midi", "midi"}, {"text/x-pascal", "pas"}, + {"text/x-asm", "asm"}, {"video/quicktime", "mov"}}; + std::size_t index = 0; + if (preffered_extensions.count(mime_type) != 0) { + index = std::find(extensions.begin(), extensions.end(), preffered_extensions[mime_type]) - extensions.begin(); + assert(index < extensions.size()); + } + if (mime_type_to_extension.emplace_hint(mime_type_to_extension.end(), mime_type, extensions[index])->second != + extensions[index]) { + std::cerr << "MIME type \"" << mime_type << "\" has more than one extensions list" << std::endl; + } + + for (auto &extension : extensions) { + if (!extension_to_mime_type.emplace(extension, mime_type).second) { + std::cerr << "Extension \"" << extension << "\" matches more than one type" << std::endl; + } + } + } + + if (!generate(argv[2], "mime_type", "extension", mime_type_to_extension)) { + return EXIT_FAILURE; + } + if (!generate(argv[3], "extension", "mime_type", extension_to_mime_type)) { + return EXIT_FAILURE; + } +} diff --git a/libs/tdlib/td/tdutils/generate/mime_types.txt b/libs/tdlib/td/tdutils/generate/mime_types.txt new file mode 100644 index 0000000000..a2d4abbf29 --- /dev/null +++ b/libs/tdlib/td/tdutils/generate/mime_types.txt @@ -0,0 +1,765 @@ +application/andrew-inset ez +application/applixware aw +application/atom+xml atom +application/atomcat+xml atomcat +application/atomsvc+xml atomsvc +application/ccxml+xml ccxml +application/cdmi-capability cdmia +application/cdmi-container cdmic +application/cdmi-domain cdmid +application/cdmi-object cdmio +application/cdmi-queue cdmiq +application/cu-seeme cu +application/davmount+xml davmount +application/docbook+xml dbk +application/dssc+der dssc +application/dssc+xml xdssc +application/ecmascript ecma +application/emma+xml emma +application/epub+zip epub +application/exi exi +application/font-tdpfr pfr +application/gml+xml gml +application/gpx+xml gpx +application/gxf gxf +application/hyperstudio stk +application/inkml+xml ink inkml +application/ipfix ipfix +application/java-archive jar +application/java-serialized-object ser +application/java-vm class +application/javascript js +application/json json +application/jsonml+json jsonml +application/lost+xml lostxml +application/mac-binhex40 hqx +application/mac-compactpro cpt +application/mads+xml mads +application/marc mrc +application/marcxml+xml mrcx +application/mathematica ma nb mb +application/mathml+xml mathml +application/mbox mbox +application/mediaservercontrol+xml mscml +application/metalink+xml metalink +application/metalink4+xml meta4 +application/mets+xml mets +application/mods+xml mods +application/mp21 m21 mp21 +application/mp4 mp4s +application/msword doc dot +application/mxf mxf +application/octet-stream bin dms lrf mar so dist distz pkg bpk dump elc deploy +application/oda oda +application/oebps-package+xml opf +application/ogg ogx +application/omdoc+xml omdoc +application/onenote onetoc onetoc2 onetmp onepkg +application/oxps oxps +application/patch-ops-error+xml xer +application/pdf pdf +application/pgp-encrypted pgp +application/pgp-signature asc sig +application/pics-rules prf +application/pkcs10 p10 +application/pkcs7-mime p7m p7c +application/pkcs7-signature p7s +application/pkcs8 p8 +application/pkix-attr-cert ac +application/pkix-cert cer +application/pkix-crl crl +application/pkix-pkipath pkipath +application/pkixcmp pki +application/pls+xml pls +application/postscript ai eps ps +application/prs.cww cww +application/pskc+xml pskcxml +application/rdf+xml rdf +application/reginfo+xml rif +application/relax-ng-compact-syntax rnc +application/resource-lists+xml rl +application/resource-lists-diff+xml rld +application/rls-services+xml rs +application/rpki-ghostbusters gbr +application/rpki-manifest mft +application/rpki-roa roa +application/rsd+xml rsd +application/rss+xml rss +application/rtf rtf +application/sbml+xml sbml +application/scvp-cv-request scq +application/scvp-cv-response scs +application/scvp-vp-request spq +application/scvp-vp-response spp +application/sdp sdp +application/set-payment-initiation setpay +application/set-registration-initiation setreg +application/shf+xml shf +application/smil+xml smi smil +application/sparql-query rq +application/sparql-results+xml srx +application/srgs gram +application/srgs+xml grxml +application/sru+xml sru +application/ssdl+xml ssdl +application/ssml+xml ssml +application/tei+xml tei teicorpus +application/thraud+xml tfi +application/timestamped-data tsd +application/vnd.3gpp.pic-bw-large plb +application/vnd.3gpp.pic-bw-small psb +application/vnd.3gpp.pic-bw-var pvb +application/vnd.3gpp2.tcap tcap +application/vnd.3m.post-it-notes pwn +application/vnd.accpac.simply.aso aso +application/vnd.accpac.simply.imp imp +application/vnd.acucobol acu +application/vnd.acucorp atc acutc +application/vnd.adobe.air-application-installer-package+zip air +application/vnd.adobe.formscentral.fcdt fcdt +application/vnd.adobe.fxp fxp fxpl +application/vnd.adobe.xdp+xml xdp +application/vnd.adobe.xfdf xfdf +application/vnd.ahead.space ahead +application/vnd.airzip.filesecure.azf azf +application/vnd.airzip.filesecure.azs azs +application/vnd.amazon.ebook azw +application/vnd.americandynamics.acc acc +application/vnd.amiga.ami ami +application/vnd.android.package-archive apk +application/vnd.anser-web-certificate-issue-initiation cii +application/vnd.anser-web-funds-transfer-initiation fti +application/vnd.antix.game-component atx +application/vnd.apple.installer+xml mpkg +application/vnd.apple.mpegurl m3u8 +application/vnd.aristanetworks.swi swi +application/vnd.astraea-software.iota iota +application/vnd.audiograph aep +application/vnd.blueice.multipass mpm +application/vnd.bmi bmi +application/vnd.businessobjects rep +application/vnd.chemdraw+xml cdxml +application/vnd.chipnuts.karaoke-mmd mmd +application/vnd.cinderella cdy +application/vnd.claymore cla +application/vnd.cloanto.rp9 rp9 +application/vnd.clonk.c4group c4g c4d c4f c4p c4u +application/vnd.cluetrust.cartomobile-config c11amc +application/vnd.cluetrust.cartomobile-config-pkg c11amz +application/vnd.commonspace csp +application/vnd.contact.cmsg cdbcmsg +application/vnd.cosmocaller cmc +application/vnd.crick.clicker clkx +application/vnd.crick.clicker.keyboard clkk +application/vnd.crick.clicker.palette clkp +application/vnd.crick.clicker.template clkt +application/vnd.crick.clicker.wordbank clkw +application/vnd.criticaltools.wbs+xml wbs +application/vnd.ctc-posml pml +application/vnd.cups-ppd ppd +application/vnd.curl.car car +application/vnd.curl.pcurl pcurl +application/vnd.dart dart +application/vnd.data-vision.rdz rdz +application/vnd.dece.data uvf uvvf uvd uvvd +application/vnd.dece.ttml+xml uvt uvvt +application/vnd.dece.unspecified uvx uvvx +application/vnd.dece.zip uvz uvvz +application/vnd.denovo.fcselayout-link fe_launch +application/vnd.dna dna +application/vnd.dolby.mlp mlp +application/vnd.dpgraph dpg +application/vnd.dreamfactory dfac +application/vnd.ds-keypoint kpxx +application/vnd.dvb.ait ait +application/vnd.dvb.service svc +application/vnd.dynageo geo +application/vnd.ecowin.chart mag +application/vnd.enliven nml +application/vnd.epson.esf esf +application/vnd.epson.msf msf +application/vnd.epson.quickanime qam +application/vnd.epson.salt slt +application/vnd.epson.ssf ssf +application/vnd.eszigno3+xml es3 et3 +application/vnd.ezpix-album ez2 +application/vnd.ezpix-package ez3 +application/vnd.fdf fdf +application/vnd.fdsn.mseed mseed +application/vnd.fdsn.seed seed dataless +application/vnd.flographit gph +application/vnd.fluxtime.clip ftc +application/vnd.framemaker fm frame maker book +application/vnd.frogans.fnc fnc +application/vnd.frogans.ltf ltf +application/vnd.fsc.weblaunch fsc +application/vnd.fujitsu.oasys oas +application/vnd.fujitsu.oasys2 oa2 +application/vnd.fujitsu.oasys3 oa3 +application/vnd.fujitsu.oasysgp fg5 +application/vnd.fujitsu.oasysprs bh2 +application/vnd.fujixerox.ddd ddd +application/vnd.fujixerox.docuworks xdw +application/vnd.fujixerox.docuworks.binder xbd +application/vnd.fuzzysheet fzs +application/vnd.genomatix.tuxedo txd +application/vnd.geogebra.file ggb +application/vnd.geogebra.tool ggt +application/vnd.geometry-explorer gex gre +application/vnd.geonext gxt +application/vnd.geoplan g2w +application/vnd.geospace g3w +application/vnd.gmx gmx +application/vnd.google-earth.kml+xml kml +application/vnd.google-earth.kmz kmz +application/vnd.grafeq gqf gqs +application/vnd.groove-account gac +application/vnd.groove-help ghf +application/vnd.groove-identity-message gim +application/vnd.groove-injector grv +application/vnd.groove-tool-message gtm +application/vnd.groove-tool-template tpl +application/vnd.groove-vcard vcg +application/vnd.hal+xml hal +application/vnd.handheld-entertainment+xml zmm +application/vnd.hbci hbci +application/vnd.hhe.lesson-player les +application/vnd.hp-hpgl hpgl +application/vnd.hp-hpid hpid +application/vnd.hp-hps hps +application/vnd.hp-jlyt jlt +application/vnd.hp-pcl pcl +application/vnd.hp-pclxl pclxl +application/vnd.hydrostatix.sof-data sfd-hdstx +application/vnd.ibm.minipay mpy +application/vnd.ibm.modcap afp listafp list3820 +application/vnd.ibm.rights-management irm +application/vnd.ibm.secure-container sc +application/vnd.iccprofile icc icm +application/vnd.igloader igl +application/vnd.immervision-ivp ivp +application/vnd.immervision-ivu ivu +application/vnd.insors.igm igm +application/vnd.intercon.formnet xpw xpx +application/vnd.intergeo i2g +application/vnd.intu.qbo qbo +application/vnd.intu.qfx qfx +application/vnd.ipunplugged.rcprofile rcprofile +application/vnd.irepository.package+xml irp +application/vnd.is-xpr xpr +application/vnd.isac.fcs fcs +application/vnd.jam jam +application/vnd.jcp.javame.midlet-rms rms +application/vnd.jisp jisp +application/vnd.joost.joda-archive joda +application/vnd.kahootz ktz ktr +application/vnd.kde.karbon karbon +application/vnd.kde.kchart chrt +application/vnd.kde.kformula kfo +application/vnd.kde.kivio flw +application/vnd.kde.kontour kon +application/vnd.kde.kpresenter kpr kpt +application/vnd.kde.kspread ksp +application/vnd.kde.kword kwd kwt +application/vnd.kenameaapp htke +application/vnd.kidspiration kia +application/vnd.kinar kne knp +application/vnd.koan skp skd skt skm +application/vnd.kodak-descriptor sse +application/vnd.las.las+xml lasxml +application/vnd.llamagraphics.life-balance.desktop lbd +application/vnd.llamagraphics.life-balance.exchange+xml lbe +application/vnd.lotus-1-2-3 123 +application/vnd.lotus-approach apr +application/vnd.lotus-freelance pre +application/vnd.lotus-notes nsf +application/vnd.lotus-organizer org +application/vnd.lotus-screencam scm +application/vnd.lotus-wordpro lwp +application/vnd.macports.portpkg portpkg +application/vnd.mcd mcd +application/vnd.medcalcdata mc1 +application/vnd.mediastation.cdkey cdkey +application/vnd.mfer mwf +application/vnd.mfmp mfm +application/vnd.micrografx.flo flo +application/vnd.micrografx.igx igx +application/vnd.mif mif +application/vnd.mobius.daf daf +application/vnd.mobius.dis dis +application/vnd.mobius.mbk mbk +application/vnd.mobius.mqy mqy +application/vnd.mobius.msl msl +application/vnd.mobius.plc plc +application/vnd.mobius.txf txf +application/vnd.mophun.application mpn +application/vnd.mophun.certificate mpc +application/vnd.mozilla.xul+xml xul +application/vnd.ms-artgalry cil +application/vnd.ms-cab-compressed cab +application/vnd.ms-excel xls xlm xla xlc xlt xlw +application/vnd.ms-excel.addin.macroenabled.12 xlam +application/vnd.ms-excel.sheet.binary.macroenabled.12 xlsb +application/vnd.ms-excel.sheet.macroenabled.12 xlsm +application/vnd.ms-excel.template.macroenabled.12 xltm +application/vnd.ms-fontobject eot +application/vnd.ms-htmlhelp chm +application/vnd.ms-ims ims +application/vnd.ms-lrm lrm +application/vnd.ms-officetheme thmx +application/vnd.ms-pki.seccat cat +application/vnd.ms-pki.stl stl +application/vnd.ms-powerpoint ppt pps pot +application/vnd.ms-powerpoint.addin.macroenabled.12 ppam +application/vnd.ms-powerpoint.presentation.macroenabled.12 pptm +application/vnd.ms-powerpoint.slide.macroenabled.12 sldm +application/vnd.ms-powerpoint.slideshow.macroenabled.12 ppsm +application/vnd.ms-powerpoint.template.macroenabled.12 potm +application/vnd.ms-project mpp mpt +application/vnd.ms-word.document.macroenabled.12 docm +application/vnd.ms-word.template.macroenabled.12 dotm +application/vnd.ms-works wps wks wcm wdb +application/vnd.ms-wpl wpl +application/vnd.ms-xpsdocument xps +application/vnd.mseq mseq +application/vnd.musician mus +application/vnd.muvee.style msty +application/vnd.mynfc taglet +application/vnd.neurolanguage.nlu nlu +application/vnd.nitf ntf nitf +application/vnd.noblenet-directory nnd +application/vnd.noblenet-sealer nns +application/vnd.noblenet-web nnw +application/vnd.nokia.n-gage.data ngdat +application/vnd.nokia.n-gage.symbian.install n-gage +application/vnd.nokia.radio-preset rpst +application/vnd.nokia.radio-presets rpss +application/vnd.novadigm.edm edm +application/vnd.novadigm.edx edx +application/vnd.novadigm.ext ext +application/vnd.oasis.opendocument.chart odc +application/vnd.oasis.opendocument.chart-template otc +application/vnd.oasis.opendocument.database odb +application/vnd.oasis.opendocument.formula odf +application/vnd.oasis.opendocument.formula-template odft +application/vnd.oasis.opendocument.graphics odg +application/vnd.oasis.opendocument.graphics-template otg +application/vnd.oasis.opendocument.image odi +application/vnd.oasis.opendocument.image-template oti +application/vnd.oasis.opendocument.presentation odp +application/vnd.oasis.opendocument.presentation-template otp +application/vnd.oasis.opendocument.spreadsheet ods +application/vnd.oasis.opendocument.spreadsheet-template ots +application/vnd.oasis.opendocument.text odt +application/vnd.oasis.opendocument.text-master odm +application/vnd.oasis.opendocument.text-template ott +application/vnd.oasis.opendocument.text-web oth +application/vnd.olpc-sugar xo +application/vnd.oma.dd2+xml dd2 +application/vnd.openofficeorg.extension oxt +application/vnd.openxmlformats-officedocument.presentationml.presentation pptx +application/vnd.openxmlformats-officedocument.presentationml.slide sldx +application/vnd.openxmlformats-officedocument.presentationml.slideshow ppsx +application/vnd.openxmlformats-officedocument.presentationml.template potx +application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx +application/vnd.openxmlformats-officedocument.spreadsheetml.template xltx +application/vnd.openxmlformats-officedocument.wordprocessingml.document docx +application/vnd.openxmlformats-officedocument.wordprocessingml.template dotx +application/vnd.osgeo.mapguide.package mgp +application/vnd.osgi.dp dp +application/vnd.osgi.subsystem esa +application/vnd.palm pdb pqa oprc +application/vnd.pawaafile paw +application/vnd.pg.format str +application/vnd.pg.osasli ei6 +application/vnd.picsel efif +application/vnd.pmi.widget wg +application/vnd.pocketlearn plf +application/vnd.powerbuilder6 pbd +application/vnd.previewsystems.box box +application/vnd.proteus.magazine mgz +application/vnd.publishare-delta-tree qps +application/vnd.pvi.ptid1 ptid +application/vnd.quark.quarkxpress qxd qxt qwd qwt qxl qxb +application/vnd.realvnc.bed bed +application/vnd.recordare.musicxml mxl +application/vnd.recordare.musicxml+xml musicxml +application/vnd.rig.cryptonote cryptonote +application/vnd.rim.cod cod +application/vnd.rn-realmedia rm +application/vnd.rn-realmedia-vbr rmvb +application/vnd.route66.link66+xml link66 +application/vnd.sailingtracker.track st +application/vnd.seemail see +application/vnd.sema sema +application/vnd.semd semd +application/vnd.semf semf +application/vnd.shana.informed.formdata ifm +application/vnd.shana.informed.formtemplate itp +application/vnd.shana.informed.interchange iif +application/vnd.shana.informed.package ipk +application/vnd.simtech-mindmapper twd twds +application/vnd.smaf mmf +application/vnd.smart.teacher teacher +application/vnd.solent.sdkm+xml sdkm sdkd +application/vnd.spotfire.dxp dxp +application/vnd.spotfire.sfs sfs +application/vnd.stardivision.calc sdc +application/vnd.stardivision.draw sda +application/vnd.stardivision.impress sdd +application/vnd.stardivision.math smf +application/vnd.stardivision.writer sdw vor +application/vnd.stardivision.writer-global sgl +application/vnd.stepmania.package smzip +application/vnd.stepmania.stepchart sm +application/vnd.sun.xml.calc sxc +application/vnd.sun.xml.calc.template stc +application/vnd.sun.xml.draw sxd +application/vnd.sun.xml.draw.template std +application/vnd.sun.xml.impress sxi +application/vnd.sun.xml.impress.template sti +application/vnd.sun.xml.math sxm +application/vnd.sun.xml.writer sxw +application/vnd.sun.xml.writer.global sxg +application/vnd.sun.xml.writer.template stw +application/vnd.sus-calendar sus susp +application/vnd.svd svd +application/vnd.symbian.install sis sisx +application/vnd.syncml+xml xsm +application/vnd.syncml.dm+wbxml bdm +application/vnd.syncml.dm+xml xdm +application/vnd.tao.intent-module-archive tao +application/vnd.tcpdump.pcap pcap cap dmp +application/vnd.tmobile-livetv tmo +application/vnd.trid.tpt tpt +application/vnd.triscape.mxs mxs +application/vnd.trueapp tra +application/vnd.ufdl ufd ufdl +application/vnd.uiq.theme utz +application/vnd.umajin umj +application/vnd.unity unityweb +application/vnd.uoml+xml uoml +application/vnd.vcx vcx +application/vnd.visio vsd vst vss vsw +application/vnd.visionary vis +application/vnd.vsf vsf +application/vnd.wap.wbxml wbxml +application/vnd.wap.wmlc wmlc +application/vnd.wap.wmlscriptc wmlsc +application/vnd.webturbo wtb +application/vnd.wolfram.player nbp +application/vnd.wordperfect wpd +application/vnd.wqd wqd +application/vnd.wt.stf stf +application/vnd.xara xar +application/vnd.xfdl xfdl +application/vnd.yamaha.hv-dic hvd +application/vnd.yamaha.hv-script hvs +application/vnd.yamaha.hv-voice hvp +application/vnd.yamaha.openscoreformat osf +application/vnd.yamaha.openscoreformat.osfpvg+xml osfpvg +application/vnd.yamaha.smaf-audio saf +application/vnd.yamaha.smaf-phrase spf +application/vnd.yellowriver-custom-menu cmp +application/vnd.zul zir zirz +application/vnd.zzazz.deck+xml zaz +application/voicexml+xml vxml +application/widget wgt +application/winhlp hlp +application/wsdl+xml wsdl +application/wspolicy+xml wspolicy +application/x-7z-compressed 7z +application/x-abiword abw +application/x-ace-compressed ace +application/x-apple-diskimage dmg +application/x-authorware-bin aab x32 u32 vox +application/x-authorware-map aam +application/x-authorware-seg aas +application/x-bcpio bcpio +application/x-bittorrent torrent +application/x-blorb blb blorb +application/x-bzip bz +application/x-bzip2 bz2 boz +application/x-cbr cbr cba cbt cbz cb7 +application/x-cdlink vcd +application/x-cfs-compressed cfs +application/x-chat chat +application/x-chess-pgn pgn +application/x-conference nsc +application/x-cpio cpio +application/x-csh csh +application/x-debian-package deb udeb +application/x-dgc-compressed dgc +application/x-director dir dcr dxr cst cct cxt w3d fgd swa +application/x-doom wad +application/x-dtbncx+xml ncx +application/x-dtbook+xml dtb +application/x-dtbresource+xml res +application/x-dvi dvi +application/x-envoy evy +application/x-eva eva +application/x-font-bdf bdf +application/x-font-ghostscript gsf +application/x-font-linux-psf psf +application/x-font-otf otf +application/x-font-pcf pcf +application/x-font-snf snf +application/x-font-ttf ttf ttc +application/x-font-type1 pfa pfb pfm afm +application/font-woff woff +application/x-freearc arc +application/x-futuresplash spl +application/x-gca-compressed gca +application/x-glulx ulx +application/x-gnumeric gnumeric +application/x-gramps-xml gramps +application/x-gtar gtar +application/x-hdf hdf +application/x-install-instructions install +application/x-iso9660-image iso +application/x-java-jnlp-file jnlp +application/x-latex latex +application/x-lzh-compressed lzh lha +application/x-mie mie +application/x-mobipocket-ebook prc mobi +application/x-ms-application application +application/x-ms-shortcut lnk +application/x-ms-wmd wmd +application/x-ms-wmz wmz +application/x-ms-xbap xbap +application/x-msaccess mdb +application/x-msbinder obd +application/x-mscardfile crd +application/x-msclip clp +application/x-msdownload exe dll com bat msi +application/x-msmediaview mvb m13 m14 +application/x-msmetafile wmf wmz emf emz +application/x-msmoney mny +application/x-mspublisher pub +application/x-msschedule scd +application/x-msterminal trm +application/x-mswrite wri +application/x-netcdf nc cdf +application/x-nzb nzb +application/x-pkcs12 p12 pfx +application/x-pkcs7-certificates p7b spc +application/x-pkcs7-certreqresp p7r +application/x-rar-compressed rar +application/x-research-info-systems ris +application/x-sh sh +application/x-shar shar +application/x-shockwave-flash swf +application/x-silverlight-app xap +application/x-sql sql +application/x-stuffit sit +application/x-stuffitx sitx +application/x-subrip srt +application/x-sv4cpio sv4cpio +application/x-sv4crc sv4crc +application/x-t3vm-image t3 +application/x-tads gam +application/x-tar tar +application/x-tcl tcl +application/x-tex tex +application/x-tex-tfm tfm +application/x-texinfo texinfo texi +application/x-tgif obj +application/x-ustar ustar +application/x-wais-source src +application/x-x509-ca-cert der crt +application/x-xfig fig +application/x-xliff+xml xlf +application/x-xpinstall xpi +application/x-xz xz +application/x-zmachine z1 z2 z3 z4 z5 z6 z7 z8 +application/xaml+xml xaml +application/xcap-diff+xml xdf +application/xenc+xml xenc +application/xhtml+xml xhtml xht +application/xml xml xsl +application/xml-dtd dtd +application/xop+xml xop +application/xproc+xml xpl +application/xslt+xml xslt +application/xspf+xml xspf +application/xv+xml mxml xhvml xvml xvm +application/yang yang +application/yin+xml yin +application/zip zip +audio/adpcm adp +audio/basic au snd +audio/midi mid midi kar rmi +audio/mp4 mp4a +audio/mpeg mpga mp2 mp2a mp3 m2a m3a +audio/ogg oga ogg spx +audio/s3m s3m +audio/silk sil +audio/vnd.dece.audio uva uvva +audio/vnd.digital-winds eol +audio/vnd.dra dra +audio/vnd.dts dts +audio/vnd.dts.hd dtshd +audio/vnd.lucent.voice lvp +audio/vnd.ms-playready.media.pya pya +audio/vnd.nuera.ecelp4800 ecelp4800 +audio/vnd.nuera.ecelp7470 ecelp7470 +audio/vnd.nuera.ecelp9600 ecelp9600 +audio/vnd.rip rip +audio/webm weba +audio/x-aac aac +audio/x-aiff aif aiff aifc +audio/x-caf caf +audio/x-flac flac +audio/x-matroska mka +audio/x-mpegurl m3u +audio/x-ms-wax wax +audio/x-ms-wma wma +audio/x-pn-realaudio ram ra +audio/x-pn-realaudio-plugin rmp +audio/x-wav wav +audio/xm xm +chemical/x-cdx cdx +chemical/x-cif cif +chemical/x-cmdf cmdf +chemical/x-cml cml +chemical/x-csml csml +chemical/x-xyz xyz +image/bmp bmp +image/cgm cgm +image/g3fax g3 +image/gif gif +image/ief ief +image/jpeg jpeg jpg jpe +image/ktx ktx +image/png png +image/prs.btif btif +image/sgi sgi +image/svg+xml svg svgz +image/tiff tiff tif +image/vnd.adobe.photoshop psd +image/vnd.dece.graphic uvi uvvi uvg uvvg +image/vnd.dvb.subtitle sub +image/vnd.djvu djvu djv +image/vnd.dwg dwg +image/vnd.dxf dxf +image/vnd.fastbidsheet fbs +image/vnd.fpx fpx +image/vnd.fst fst +image/vnd.fujixerox.edmics-mmr mmr +image/vnd.fujixerox.edmics-rlc rlc +image/vnd.ms-modi mdi +image/vnd.ms-photo wdp +image/vnd.net-fpx npx +image/vnd.wap.wbmp wbmp +image/vnd.xiff xif +image/webp webp +image/x-3ds 3ds +image/x-cmu-raster ras +image/x-cmx cmx +image/x-freehand fh fhc fh4 fh5 fh7 +image/x-icon ico +image/x-mrsid-image sid +image/x-pcx pcx +image/x-pict pic pct +image/x-portable-anymap pnm +image/x-portable-bitmap pbm +image/x-portable-graymap pgm +image/x-portable-pixmap ppm +image/x-rgb rgb +image/x-tga tga +image/x-xbitmap xbm +image/x-xpixmap xpm +image/x-xwindowdump xwd +message/rfc822 eml mime +model/iges igs iges +model/mesh msh mesh silo +model/vnd.collada+xml dae +model/vnd.dwf dwf +model/vnd.gdl gdl +model/vnd.gtw gtw +model/vnd.mts mts +model/vnd.vtu vtu +model/vrml wrl vrml +model/x3d+binary x3db x3dbz +model/x3d+vrml x3dv x3dvz +model/x3d+xml x3d x3dz +text/cache-manifest appcache +text/calendar ics ifb +text/css css +text/csv csv +text/html html htm +text/n3 n3 +text/plain txt text conf def list log in +text/prs.lines.tag dsc +text/richtext rtx +text/sgml sgml sgm +text/tab-separated-values tsv +text/troff t tr roff man me ms +text/turtle ttl +text/uri-list uri uris urls +text/vcard vcard +text/vnd.curl curl +text/vnd.curl.dcurl dcurl +text/vnd.curl.scurl scurl +text/vnd.curl.mcurl mcurl +text/vnd.dvb.subtitle sub +text/vnd.fly fly +text/vnd.fmi.flexstor flx +text/vnd.graphviz gv +text/vnd.in3d.3dml 3dml +text/vnd.in3d.spot spot +text/vnd.sun.j2me.app-descriptor jad +text/vnd.wap.wml wml +text/vnd.wap.wmlscript wmls +text/x-asm s asm +text/x-c c cc cxx cpp h hh dic +text/x-fortran f for f77 f90 +text/x-java-source java +text/x-opml opml +text/x-pascal p pas +text/x-nfo nfo +text/x-setext etx +text/x-sfv sfv +text/x-uuencode uu +text/x-vcalendar vcs +text/x-vcard vcf +video/3gpp 3gp +video/3gpp2 3g2 +video/h261 h261 +video/h263 h263 +video/h264 h264 +video/jpeg jpgv +video/jpm jpm jpgm +video/mj2 mj2 mjp2 +video/mp4 mp4 mp4v mpg4 +video/mpeg mpeg mpg mpe m1v m2v +video/ogg ogv +video/quicktime qt mov +video/vnd.dece.hd uvh uvvh +video/vnd.dece.mobile uvm uvvm +video/vnd.dece.pd uvp uvvp +video/vnd.dece.sd uvs uvvs +video/vnd.dece.video uvv uvvv +video/vnd.dvb.file dvb +video/vnd.fvt fvt +video/vnd.mpegurl mxu m4u +video/vnd.ms-playready.media.pyv pyv +video/vnd.uvvu.mp4 uvu uvvu +video/vnd.vivo viv +video/webm webm +video/x-f4v f4v +video/x-fli fli +video/x-flv flv +video/x-m4v m4v +video/x-matroska mkv mk3d mks +video/x-mng mng +video/x-ms-asf asf asx +video/x-ms-vob vob +video/x-ms-wm wm +video/x-ms-wmv wmv +video/x-ms-wmx wmx +video/x-ms-wvx wvx +video/x-msvideo avi +video/x-sgi-movie movie +video/x-smv smv +x-conference/x-cooltalk ice diff --git a/libs/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h b/libs/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h new file mode 100644 index 0000000000..820aa02fbe --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/AesCtrByteFlow.h @@ -0,0 +1,55 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/ByteFlow.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +#if TD_HAVE_OPENSSL +class AesCtrByteFlow : public ByteFlowInplaceBase { + public: + void init(const UInt256 &key, const UInt128 &iv) { + state_.init(key, iv); + } + void init(AesCtrState &&state) { + state_ = std::move(state); + } + AesCtrState move_aes_ctr_state() { + return std::move(state_); + } + void loop() override { + bool was_updated = false; + while (true) { + auto ready = input_->prepare_read(); + if (ready.empty()) { + break; + } + state_.encrypt(ready, MutableSlice(const_cast<char *>(ready.data()), ready.size())); + input_->confirm_read(ready.size()); + output_.advance_end(ready.size()); + was_updated = true; + } + if (was_updated) { + on_output_updated(); + } + if (!is_input_active_) { + finish(Status::OK()); // End of input stream. + } + set_need_size(1); + } + + private: + AesCtrState state_; +}; +#endif + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/BigNum.cpp b/libs/tdlib/td/tdutils/td/utils/BigNum.cpp new file mode 100644 index 0000000000..f553661d49 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/BigNum.cpp @@ -0,0 +1,251 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/BigNum.h" + +char disable_linker_warning_about_empty_file_bignum_cpp TD_UNUSED; + +#if TD_HAVE_OPENSSL + +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +#include <openssl/bn.h> +#include <openssl/crypto.h> + +namespace td { + +class BigNumContext::Impl { + public: + BN_CTX *big_num_context; + + Impl() : big_num_context(BN_CTX_new()) { + LOG_IF(FATAL, big_num_context == nullptr); + } + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl() { + BN_CTX_free(big_num_context); + } +}; + +BigNumContext::BigNumContext() : impl_(make_unique<Impl>()) { +} + +BigNumContext::BigNumContext(BigNumContext &&other) = default; +BigNumContext &BigNumContext::operator=(BigNumContext &&other) = default; + +BigNumContext::~BigNumContext() = default; + +class BigNum::Impl { + public: + BIGNUM *big_num; + + Impl() : Impl(BN_new()) { + } + explicit Impl(BIGNUM *big_num) : big_num(big_num) { + LOG_IF(FATAL, big_num == nullptr); + } + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl() { + BN_clear_free(big_num); + } +}; + +BigNum::BigNum() : impl_(make_unique<Impl>()) { +} + +BigNum::BigNum(const BigNum &other) : BigNum() { + *this = other; +} + +BigNum &BigNum::operator=(const BigNum &other) { + CHECK(impl_ != nullptr); + CHECK(other.impl_ != nullptr); + BIGNUM *result = BN_copy(impl_->big_num, other.impl_->big_num); + LOG_IF(FATAL, result == nullptr); + return *this; +} + +BigNum::BigNum(BigNum &&other) = default; + +BigNum &BigNum::operator=(BigNum &&other) = default; + +BigNum::~BigNum() = default; + +BigNum BigNum::from_binary(Slice str) { + return BigNum(make_unique<Impl>(BN_bin2bn(str.ubegin(), narrow_cast<int>(str.size()), nullptr))); +} + +BigNum BigNum::from_decimal(CSlice str) { + BigNum result; + int err = BN_dec2bn(&result.impl_->big_num, str.c_str()); + LOG_IF(FATAL, err == 0); + return result; +} + +BigNum BigNum::from_raw(void *openssl_big_num) { + return BigNum(make_unique<Impl>(static_cast<BIGNUM *>(openssl_big_num))); +} + +BigNum::BigNum(unique_ptr<Impl> &&impl) : impl_(std::move(impl)) { +} + +void BigNum::ensure_const_time() { + BN_set_flags(impl_->big_num, BN_FLG_CONSTTIME); +} + +int BigNum::get_num_bits() const { + return BN_num_bits(impl_->big_num); +} + +int BigNum::get_num_bytes() const { + return BN_num_bytes(impl_->big_num); +} + +void BigNum::set_bit(int num) { + int result = BN_set_bit(impl_->big_num, num); + LOG_IF(FATAL, result != 1); +} + +void BigNum::clear_bit(int num) { + int result = BN_clear_bit(impl_->big_num, num); + LOG_IF(FATAL, result != 1); +} + +bool BigNum::is_bit_set(int num) const { + return BN_is_bit_set(impl_->big_num, num) != 0; +} + +bool BigNum::is_prime(BigNumContext &context) const { + int result = BN_is_prime_ex(impl_->big_num, BN_prime_checks, context.impl_->big_num_context, nullptr); + LOG_IF(FATAL, result == -1); + return result == 1; +} + +void BigNum::operator+=(uint32 value) { + int result = BN_add_word(impl_->big_num, value); + LOG_IF(FATAL, result != 1); +} + +void BigNum::operator-=(uint32 value) { + int result = BN_sub_word(impl_->big_num, value); + LOG_IF(FATAL, result != 1); +} + +void BigNum::operator*=(uint32 value) { + int result = BN_mul_word(impl_->big_num, value); + LOG_IF(FATAL, result != 1); +} + +void BigNum::operator/=(uint32 value) { + BN_ULONG result = BN_div_word(impl_->big_num, value); + LOG_IF(FATAL, result == static_cast<BN_ULONG>(-1)); +} + +uint32 BigNum::operator%(uint32 value) const { + BN_ULONG result = BN_mod_word(impl_->big_num, value); + LOG_IF(FATAL, result == static_cast<BN_ULONG>(-1)); + return narrow_cast<uint32>(result); +} + +void BigNum::set_value(uint32 new_value) { + if (new_value == 0) { + BN_zero(impl_->big_num); + } else { + int result = BN_set_word(impl_->big_num, new_value); + LOG_IF(FATAL, result != 1); + } +} + +BigNum BigNum::clone() const { + BIGNUM *result = BN_dup(impl_->big_num); + LOG_IF(FATAL, result == nullptr); + return BigNum(make_unique<Impl>(result)); +} + +string BigNum::to_binary(int exact_size) const { + int num_size = get_num_bytes(); + if (exact_size == -1) { + exact_size = num_size; + } else { + CHECK(exact_size >= num_size); + } + string res(exact_size, '\0'); + BN_bn2bin(impl_->big_num, reinterpret_cast<unsigned char *>(&res[exact_size - num_size])); + return res; +} + +string BigNum::to_decimal() const { + char *result = BN_bn2dec(impl_->big_num); + CHECK(result != nullptr); + string res(result); + OPENSSL_free(result); + return res; +} + +void BigNum::random(BigNum &r, int bits, int top, int bottom) { + int result = BN_rand(r.impl_->big_num, bits, top, bottom); + LOG_IF(FATAL, result != 1); +} + +void BigNum::add(BigNum &r, const BigNum &a, const BigNum &b) { + int result = BN_add(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num); + LOG_IF(FATAL, result != 1); +} + +void BigNum::sub(BigNum &r, const BigNum &a, const BigNum &b) { + CHECK(r.impl_->big_num != a.impl_->big_num); + CHECK(r.impl_->big_num != b.impl_->big_num); + int result = BN_sub(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) { + int result = BN_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context) { + int result = BN_mod_mul(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, m.impl_->big_num, + context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, + BigNumContext &context) { + auto q = quotient == nullptr ? nullptr : quotient->impl_->big_num; + auto r = remainder == nullptr ? nullptr : remainder->impl_->big_num; + if (q == nullptr && r == nullptr) { + return; + } + + auto result = BN_div(q, r, dividend.impl_->big_num, divisor.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context) { + int result = BN_mod_exp(r.impl_->big_num, a.impl_->big_num, p.impl_->big_num, m.impl_->big_num, + context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +void BigNum::gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context) { + int result = BN_gcd(r.impl_->big_num, a.impl_->big_num, b.impl_->big_num, context.impl_->big_num_context); + LOG_IF(FATAL, result != 1); +} + +int BigNum::compare(const BigNum &a, const BigNum &b) { + return BN_cmp(a.impl_->big_num, b.impl_->big_num); +} + +} // namespace td +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/BigNum.h b/libs/tdlib/td/tdutils/td/utils/BigNum.h new file mode 100644 index 0000000000..6eecdeab03 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/BigNum.h @@ -0,0 +1,108 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#if TD_HAVE_OPENSSL + +#include "td/utils/Slice.h" + +namespace td { + +class BigNumContext { + public: + BigNumContext(); + BigNumContext(const BigNumContext &other) = delete; + BigNumContext &operator=(const BigNumContext &other) = delete; + BigNumContext(BigNumContext &&other); + BigNumContext &operator=(BigNumContext &&other); + ~BigNumContext(); + + private: + class Impl; + unique_ptr<Impl> impl_; + + friend class BigNum; +}; + +class BigNum { + public: + BigNum(); + BigNum(const BigNum &other); + BigNum &operator=(const BigNum &other); + BigNum(BigNum &&other); + BigNum &operator=(BigNum &&other); + ~BigNum(); + + static BigNum from_binary(Slice str); + + static BigNum from_decimal(CSlice str); + + static BigNum from_raw(void *openssl_big_num); + + void set_value(uint32 new_value); + + void ensure_const_time(); + + int get_num_bits() const; + + int get_num_bytes() const; + + void set_bit(int num); + + void clear_bit(int num); + + bool is_bit_set(int num) const; + + bool is_prime(BigNumContext &context) const; + + BigNum clone() const; + + string to_binary(int exact_size = -1) const; + + string to_decimal() const; + + void operator+=(uint32 value); + + void operator-=(uint32 value); + + void operator*=(uint32 value); + + void operator/=(uint32 value); + + uint32 operator%(uint32 value) const; + + static void random(BigNum &r, int bits, int top, int bottom); + + static void add(BigNum &r, const BigNum &a, const BigNum &b); + + static void sub(BigNum &r, const BigNum &a, const BigNum &b); + + static void mul(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context); + + static void mod_mul(BigNum &r, BigNum &a, BigNum &b, const BigNum &m, BigNumContext &context); + + static void div(BigNum *quotient, BigNum *remainder, const BigNum ÷nd, const BigNum &divisor, + BigNumContext &context); + + static void mod_exp(BigNum &r, const BigNum &a, const BigNum &p, const BigNum &m, BigNumContext &context); + + static void gcd(BigNum &r, BigNum &a, BigNum &b, BigNumContext &context); + + static int compare(const BigNum &a, const BigNum &b); + + private: + class Impl; + unique_ptr<Impl> impl_; + + explicit BigNum(unique_ptr<Impl> &&impl); +}; + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/BufferedFd.h b/libs/tdlib/td/tdutils/td/utils/BufferedFd.h new file mode 100644 index 0000000000..0c8f65408d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/BufferedFd.h @@ -0,0 +1,199 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <limits> + +namespace td { +// just reads from given reader and writes to given writer +template <class FdT> +class BufferedFdBase : public FdT { + public: + BufferedFdBase() = default; + explicit BufferedFdBase(FdT &&fd_); + // TODO: make move constructor and move assignment safer + + Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT; + Result<size_t> flush_write() TD_WARN_UNUSED_RESULT; + + bool need_flush_write(size_t at_least = 0) { + CHECK(write_); + write_->sync_with_writer(); + return write_->size() > at_least; + } + size_t ready_for_flush_write() { + CHECK(write_); + write_->sync_with_writer(); + return write_->size(); + } + void set_input_writer(ChainBufferWriter *read) { + read_ = read; + } + void set_output_reader(ChainBufferReader *write) { + write_ = write; + } + + private: + ChainBufferWriter *read_ = nullptr; + ChainBufferReader *write_ = nullptr; +}; + +template <class FdT> +class BufferedFd : public BufferedFdBase<FdT> { + using Parent = BufferedFdBase<FdT>; + ChainBufferWriter input_writer_; + ChainBufferReader input_reader_; + ChainBufferWriter output_writer_; + ChainBufferReader output_reader_; + void init(); + void init_ptr(); + + public: + BufferedFd(); + explicit BufferedFd(FdT &&fd_); + BufferedFd(BufferedFd &&); + BufferedFd &operator=(BufferedFd &&); + BufferedFd(const BufferedFd &) = delete; + BufferedFd &operator=(const BufferedFd &) = delete; + ~BufferedFd(); + + void close(); + + Result<size_t> flush_read(size_t max_read = std::numeric_limits<size_t>::max()) TD_WARN_UNUSED_RESULT; + Result<size_t> flush_write() TD_WARN_UNUSED_RESULT; + + // Yep, direct access to buffers. It is IO interface too. + ChainBufferReader &input_buffer(); + ChainBufferWriter &output_buffer(); +}; + +// IMPLEMENTATION + +/*** BufferedFd ***/ +template <class FdT> +BufferedFdBase<FdT>::BufferedFdBase(FdT &&fd_) : FdT(std::move(fd_)) { +} + +template <class FdT> +Result<size_t> BufferedFdBase<FdT>::flush_read(size_t max_read) { + CHECK(read_); + size_t result = 0; + while (::td::can_read(*this) && max_read) { + MutableSlice slice = read_->prepare_append().truncate(max_read); + TRY_RESULT(x, FdT::read(slice)); + slice.truncate(x); + read_->confirm_append(x); + result += x; + max_read -= x; + } + return result; +} + +template <class FdT> +Result<size_t> BufferedFdBase<FdT>::flush_write() { + size_t result = 0; + // TODO: sync on demand + write_->sync_with_writer(); + while (!write_->empty() && ::td::can_write(*this)) { + Slice slice = write_->prepare_read(); + TRY_RESULT(x, FdT::write(slice)); + write_->confirm_read(x); + result += x; + } + return result; +} + +/*** BufferedFd ***/ +template <class FdT> +void BufferedFd<FdT>::init() { + input_reader_ = input_writer_.extract_reader(); + output_reader_ = output_writer_.extract_reader(); + init_ptr(); +} + +template <class FdT> +void BufferedFd<FdT>::init_ptr() { + this->set_input_writer(&input_writer_); + this->set_output_reader(&output_reader_); +} + +template <class FdT> +BufferedFd<FdT>::BufferedFd() { + init(); +} + +template <class FdT> +BufferedFd<FdT>::BufferedFd(FdT &&fd_) : Parent(std::move(fd_)) { + init(); +} + +template <class FdT> +BufferedFd<FdT>::BufferedFd(BufferedFd &&from) { + *this = std::move(from); +} + +template <class FdT> +BufferedFd<FdT> &BufferedFd<FdT>::operator=(BufferedFd &&from) { + FdT::operator=(std::move(static_cast<FdT &>(from))); + input_reader_ = std::move(from.input_reader_); + input_writer_ = std::move(from.input_writer_); + output_reader_ = std::move(from.output_reader_); + output_writer_ = std::move(from.output_writer_); + init_ptr(); + return *this; +} + +template <class FdT> +BufferedFd<FdT>::~BufferedFd() { + close(); +} + +template <class FdT> +void BufferedFd<FdT>::close() { + FdT::close(); + // TODO: clear buffers +} + +template <class FdT> +Result<size_t> BufferedFd<FdT>::flush_read(size_t max_read) { + TRY_RESULT(result, Parent::flush_read(max_read)); + if (result) { + // TODO: faster sync is possible if you owns writer. + input_reader_.sync_with_writer(); + LOG(DEBUG) << "flush_read: +" << format::as_size(result) << tag("total", format::as_size(input_reader_.size())); + } + return result; +} + +template <class FdT> +Result<size_t> BufferedFd<FdT>::flush_write() { + TRY_RESULT(result, Parent::flush_write()); + if (result) { + LOG(DEBUG) << "flush_write: +" << format::as_size(result) << tag("left", format::as_size(output_reader_.size())); + } + return result; +} + +// Yep, direct access to buffers. It is IO interface too. +template <class FdT> +ChainBufferReader &BufferedFd<FdT>::input_buffer() { + return input_reader_; +} + +template <class FdT> +ChainBufferWriter &BufferedFd<FdT>::output_buffer() { + return output_writer_; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/BufferedReader.h b/libs/tdlib/td/tdutils/td/utils/BufferedReader.h new file mode 100644 index 0000000000..9006d78132 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/BufferedReader.h @@ -0,0 +1,61 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <cstring> + +namespace td { +class BufferedReader { + public: + explciit BufferedReader(FileFd &file, size_t buff_size = 8152) + : file_(file), buff_(buff_size), begin_pos_(0), end_pos_(0) { + } + + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT; + + private: + FileFd &file_; + vector<char> buff_; + size_t begin_pos_; + size_t end_pos_; +}; + +inline Result<size_t> BufferedReader::read(MutableSlice slice) { + size_t available = end_pos_ - begin_pos_; + if (available >= slice.size()) { + // have enough data in buffer + std::memcpy(slice.begin(), &buff_[begin_pos_], slice.size()); + begin_pos_ += slice.size(); + return slice.size(); + } + + if (available) { + std::memcpy(slice.begin(), &buff_[begin_pos_], available); + begin_pos_ += available; + slice.remove_prefix(available); + } + + if (slice.size() > buff_.size() / 2) { + TRY_RESULT(result, file_.read(slice)); + return result + available; + } + + TRY_RESULT(result, file_.read({&buff_[0], buff_.size()})); + begin_pos_ = 0; + end_pos_ = result; + + size_t left = min(end_pos_, slice.size()); + std::memcpy(slice.begin(), &buff_[begin_pos_], left); + begin_pos_ += left; + return left + available; +} +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/ByteFlow.h b/libs/tdlib/td/tdutils/td/utils/ByteFlow.h new file mode 100644 index 0000000000..fb0c4489eb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/ByteFlow.h @@ -0,0 +1,288 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +namespace td { + +class ByteFlowInterface { + public: + virtual void close_input(Status status) = 0; + virtual void wakeup() = 0; + virtual void set_parent(ByteFlowInterface &other) = 0; + virtual void set_input(ChainBufferReader *input) = 0; + virtual size_t get_need_size() = 0; + ByteFlowInterface() = default; + ByteFlowInterface(const ByteFlowInterface &) = delete; + ByteFlowInterface &operator=(const ByteFlowInterface &) = delete; + ByteFlowInterface(ByteFlowInterface &&) = default; + ByteFlowInterface &operator=(ByteFlowInterface &&) = default; + virtual ~ByteFlowInterface() = default; +}; + +class ByteFlowBaseCommon : public ByteFlowInterface { + public: + ByteFlowBaseCommon() = default; + + void close_input(Status status) final { + if (status.is_error()) { + finish(std::move(status)); + } else { + is_input_active_ = false; + wakeup(); + } + } + + void wakeup() final { + if (stop_flag_) { + return; + } + input_->sync_with_writer(); + if (waiting_flag_) { + if (!is_input_active_) { + finish(Status::OK()); + } + return; + } + if (is_input_active_) { + if (need_size_ != 0 && input_->size() < need_size_) { + return; + } + } + need_size_ = 0; + loop(); + } + + size_t get_need_size() final { + return need_size_; + } + + virtual void loop() = 0; + + protected: + bool waiting_flag_ = false; + ChainBufferReader *input_; + bool is_input_active_ = true; + size_t need_size_ = 0; + void finish(Status status) { + stop_flag_ = true; + need_size_ = 0; + if (parent_) { + parent_->close_input(std::move(status)); + parent_ = nullptr; + } + } + + void set_need_size(size_t need_size) { + need_size_ = need_size; + } + + void on_output_updated() { + if (parent_) { + parent_->wakeup(); + } + } + void consume_input() { + waiting_flag_ = true; + if (!is_input_active_) { + finish(Status::OK()); + } + } + + private: + ByteFlowInterface *parent_ = nullptr; + bool stop_flag_ = false; + friend class ByteFlowBase; + friend class ByteFlowInplaceBase; +}; + +class ByteFlowBase : public ByteFlowBaseCommon { + public: + ByteFlowBase() = default; + + void set_input(ChainBufferReader *input) final { + input_ = input; + } + void set_parent(ByteFlowInterface &other) final { + parent_ = &other; + parent_->set_input(&output_reader_); + } + void loop() override = 0; + + // ChainBufferWriter &get_output() { + // return output_; + //} + + protected: + ChainBufferWriter output_; + ChainBufferReader output_reader_ = output_.extract_reader(); +}; + +class ByteFlowInplaceBase : public ByteFlowBaseCommon { + public: + ByteFlowInplaceBase() = default; + + void set_input(ChainBufferReader *input) final { + input_ = input; + output_ = ChainBufferReader(input_->begin().clone(), input_->begin().clone(), false); + } + void set_parent(ByteFlowInterface &other) final { + parent_ = &other; + parent_->set_input(&output_); + } + void loop() override = 0; + + ChainBufferReader &get_output() { + return output_; + } + + protected: + ChainBufferReader output_; +}; + +inline ByteFlowInterface &operator>>(ByteFlowInterface &from, ByteFlowInterface &to) { + from.set_parent(to); + return to; +} + +class ByteFlowSource : public ByteFlowInterface { + public: + ByteFlowSource() = default; + explicit ByteFlowSource(ChainBufferReader *buffer) : buffer_(buffer) { + } + ByteFlowSource(ByteFlowSource &&other) : buffer_(other.buffer_), parent_(other.parent_) { + other.buffer_ = nullptr; + other.parent_ = nullptr; + } + ByteFlowSource &operator=(ByteFlowSource &&other) { + buffer_ = other.buffer_; + parent_ = other.parent_; + other.buffer_ = nullptr; + other.parent_ = nullptr; + return *this; + } + ByteFlowSource(const ByteFlowSource &) = delete; + ByteFlowSource &operator=(const ByteFlowSource &) = delete; + ~ByteFlowSource() override = default; + + void set_input(ChainBufferReader *) final { + UNREACHABLE(); + } + void set_parent(ByteFlowInterface &parent) final { + CHECK(parent_ == nullptr); + parent_ = &parent; + parent_->set_input(buffer_); + } + void close_input(Status status) final { + CHECK(parent_); + parent_->close_input(std::move(status)); + parent_ = nullptr; + } + void wakeup() final { + CHECK(parent_); + parent_->wakeup(); + } + size_t get_need_size() final { + if (parent_ == nullptr) { + return 0; + } + return parent_->get_need_size(); + } + + private: + ChainBufferReader *buffer_ = nullptr; + ByteFlowInterface *parent_ = nullptr; +}; + +class ByteFlowSink : public ByteFlowInterface { + public: + void set_input(ChainBufferReader *input) final { + CHECK(buffer_ == nullptr); + buffer_ = input; + } + void set_parent(ByteFlowInterface &parent) final { + UNREACHABLE(); + } + void close_input(Status status) final { + CHECK(active_); + active_ = false; + status_ = std::move(status); + buffer_->sync_with_writer(); + } + void wakeup() final { + buffer_->sync_with_writer(); + } + size_t get_need_size() final { + UNREACHABLE(); + return 0; + } + bool is_ready() { + return !active_; + } + Status &status() { + return status_; + } + ChainBufferReader *result() { + CHECK(is_ready() && status().is_ok()); + return buffer_; + } + ChainBufferReader *get_output() { + return buffer_; + } + + private: + bool active_ = true; + Status status_; + ChainBufferReader *buffer_ = nullptr; +}; + +class ByteFlowMoveSink : public ByteFlowInterface { + public: + void set_input(ChainBufferReader *input) final { + CHECK(!input_); + input_ = input; + } + void set_parent(ByteFlowInterface &parent) final { + UNREACHABLE(); + } + void close_input(Status status) final { + CHECK(active_); + active_ = false; + status_ = std::move(status); + wakeup(); + } + void wakeup() final { + input_->sync_with_writer(); + output_->append(*input_); + } + size_t get_need_size() final { + UNREACHABLE(); + return 0; + } + void set_output(ChainBufferWriter *output) { + CHECK(!output_); + output_ = output; + } + + bool is_ready() { + return !active_; + } + Status &status() { + return status_; + } + + private: + bool active_ = true; + Status status_; + ChainBufferReader *input_ = nullptr; + ChainBufferWriter *output_ = nullptr; +}; +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/ChangesProcessor.h b/libs/tdlib/td/tdutils/td/utils/ChangesProcessor.h new file mode 100644 index 0000000000..9342f45a8b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/ChangesProcessor.h @@ -0,0 +1,61 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include <utility> + +namespace td { + +// Process changes after they are finished in order of addition +template <class DataT> +class ChangesProcessor { + public: + using Id = uint64; + + void clear() { + offset_ += data_array_.size(); + ready_i_ = 0; + data_array_.clear(); + } + + template <class FromDataT> + Id add(FromDataT &&data) { + auto res = offset_ + data_array_.size(); + data_array_.emplace_back(std::forward<DataT>(data), false); + return static_cast<Id>(res); + } + + template <class F> + void finish(Id token, F &&func) { + size_t pos = static_cast<size_t>(token) - offset_; + if (pos >= data_array_.size()) { + return; + } + data_array_[pos].second = true; + while (ready_i_ < data_array_.size() && data_array_[ready_i_].second == true) { + func(std::move(data_array_[ready_i_].first)); + ready_i_++; + } + try_compactify(); + } + + private: + size_t offset_ = 1; + size_t ready_i_ = 0; + std::vector<std::pair<DataT, bool>> data_array_; + void try_compactify() { + if (ready_i_ > 5 && ready_i_ * 2 > data_array_.size()) { + data_array_.erase(data_array_.begin(), data_array_.begin() + ready_i_); + offset_ += ready_i_; + ready_i_ = 0; + } + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Closure.h b/libs/tdlib/td/tdutils/td/utils/Closure.h new file mode 100644 index 0000000000..718f930b8a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Closure.h @@ -0,0 +1,169 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/logging.h" + +#include <cstdlib> +#include <tuple> +#include <type_traits> +#include <utility> + +// +// Essentially we have: +// (ActorT::func, arg1, arg2, ..., argn) +// We want to call: +// actor->func(arg1, arg2, ..., argn) +// And in some cases we would like to delay this call. +// +// First attempt would be +// [a1=arg1, a2=arg2, ..., an=argn](ActorT *actor) { +// actor->func(a1, a2, ..., an) +// } +// +// But there are some difficulties with elimitation on unnecessary copies. +// We want to use move constructor when it is possible +// +// We may pass +// Tmp. Temporary / rvalue reference +// Var. Variable / reference +// CnstRef. const reference +// +// +// Function may expect +// Val. Value +// CnstRef. const reference +// Ref. rvalue reverence / reference +// +// TODO: +// Immediate call / Delayed call +// Tmp->Val move / move->move +// Tmp->CnstRef + / move->+ +// Tmp->Ref + / move->+ +// Var->Val copy / copy->move +// Var->CnstRef + / copy-> +// Var->Ref + / copy->+ // khm. It will complile, but won't work +// +// So I will use common idiom: forward references +// If delay is needed, just std::forward data to temporary storage, and std::move them when call is executed. +// +// +// create_immediate_closure(&Actor::func, arg1, arg2, ..., argn).run(actor) +// to_delayed_closure(std::move(immediate)).run(actor) + +namespace td { +template <class ActorT, class FunctionT, class... ArgsT> +class DelayedClosure; + +template <class ActorT, class FunctionT, class... ArgsT> +class ImmediateClosure { + public: + using Delayed = DelayedClosure<ActorT, FunctionT, ArgsT...>; + friend Delayed; + using ActorType = ActorT; + + void run(ActorT *actor) { + mem_call_tuple(actor, func, std::move(args)); + } + + // no &&. just save references as references. + explicit ImmediateClosure(FunctionT func, ArgsT... args) : func(func), args(std::forward<ArgsT>(args)...) { + } + + private: + FunctionT func; + std::tuple<ArgsT...> args; +}; + +template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT> +ImmediateClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...> create_immediate_closure( + ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { + return ImmediateClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...>(func, + std::forward<SrcArgsT>(args)...); +} + +template <class ActorT, class FunctionT, class... ArgsT> +class DelayedClosure { + public: + using ActorType = ActorT; + using Delayed = DelayedClosure<ActorT, FunctionT, ArgsT...>; + + void run(ActorT *actor) { + mem_call_tuple(actor, func, std::move(args)); + } + + DelayedClosure clone() const { + return do_clone(*this); + } + + explicit DelayedClosure(ImmediateClosure<ActorT, FunctionT, ArgsT...> &&other) + : func(std::move(other.func)), args(std::move(other.args)) { + } + + explicit DelayedClosure(FunctionT func, ArgsT... args) : func(func), args(std::forward<ArgsT>(args)...) { + } + + template <class F> + void for_each(const F &f) { + tuple_for_each(args, f); + } + + private: + using ArgsStorageT = std::tuple<typename std::decay<ArgsT>::type...>; + + FunctionT func; + ArgsStorageT args; + + template <class FromActorT, class FromFunctionT, class... FromArgsT> + explicit DelayedClosure(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &other, + std::enable_if_t<LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value, int> = 0) + : func(other.func), args(other.args) { + } + + template <class FromActorT, class FromFunctionT, class... FromArgsT> + explicit DelayedClosure( + const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &other, + std::enable_if_t<!LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value, int> = 0) { + LOG(FATAL) << "Deleted constructor"; + std::abort(); + } + + template <class FromActorT, class FromFunctionT, class... FromArgsT> + std::enable_if_t<!LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value, + DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>> + do_clone(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &value) const { + LOG(FATAL) << "Trying to clone DelayedClosure that contains noncopyable elements"; + std::abort(); + } + + template <class FromActorT, class FromFunctionT, class... FromArgsT> + std::enable_if_t<LogicAnd<std::is_copy_constructible<FromArgsT>::value...>::value, + DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>> + do_clone(const DelayedClosure<FromActorT, FromFunctionT, FromArgsT...> &value) const { + return DelayedClosure<FromActorT, FromFunctionT, FromArgsT...>(value); + } +}; + +template <class... ArgsT> +typename ImmediateClosure<ArgsT...>::Delayed to_delayed_closure(ImmediateClosure<ArgsT...> &&other) { + return typename ImmediateClosure<ArgsT...>::Delayed(std::move(other)); +} + +template <class... ArgsT> +DelayedClosure<ArgsT...> to_delayed_closure(DelayedClosure<ArgsT...> &&other) { + return std::move(other); +} + +template <class ActorT, class ResultT, class... DestArgsT, class... SrcArgsT> +auto create_delayed_closure(ResultT (ActorT::*func)(DestArgsT...), SrcArgsT &&... args) { + return DelayedClosure<ActorT, ResultT (ActorT::*)(DestArgsT...), SrcArgsT &&...>(func, + std::forward<SrcArgsT>(args)...); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Container.h b/libs/tdlib/td/tdutils/td/utils/Container.h new file mode 100644 index 0000000000..57b4bb4d16 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Container.h @@ -0,0 +1,149 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <limits> + +namespace td { + +// 1. Allocates all objects in vector. (but vector never shrinks) +// 2. Id is safe way to reach this object. +// 3. All ids are unique. +// 4. All ids are non-zero. +template <class DataT> +class Container { + public: + using Id = uint64; + DataT *get(Id id) { + int32 slot_id = decode_id(id); + if (slot_id == -1) { + return nullptr; + } + return &slots_[slot_id].data; + } + + void erase(Id id) { + int32 slot_id = decode_id(id); + if (slot_id == -1) { + return; + } + release(slot_id); + } + + DataT extract(Id id) { + int32 slot_id = decode_id(id); + CHECK(slot_id != -1); + auto res = std::move(slots_[slot_id].data); + release(slot_id); + return res; + } + + Id create(DataT &&data = DataT(), uint8 type = 0) { + int32 id = store(std::move(data), type); + return encode_id(id); + } + + Id reset_id(Id id) { + int32 slot_id = decode_id(id); + CHECK(slot_id != -1); + inc_generation(slot_id); + return encode_id(slot_id); + } + + static uint8 type_from_id(Id id) { + return static_cast<uint8>(id); + } + + vector<Id> ids() { + vector<bool> is_bad(slots_.size(), false); + for (auto id : empty_slots_) { + is_bad[id] = true; + } + vector<Id> res; + for (size_t i = 0, n = slots_.size(); i < n; i++) { + if (!is_bad[i]) { + res.push_back(encode_id(static_cast<int32>(i))); + } + } + return res; + } + template <class F> + void for_each(const F &f) { + auto ids = this->ids(); + for (auto id : ids) { + f(id, *get(id)); + } + } + size_t size() const { + CHECK(empty_slots_.size() <= slots_.size()); + return slots_.size() - empty_slots_.size(); + } + bool empty() const { + return size() == 0; + } + void clear() { + *this = Container<DataT>(); + } + + private: + static constexpr uint32 GENERATION_STEP = 1 << 8; + static constexpr uint32 TYPE_MASK = (1 << 8) - 1; + struct Slot { + uint32 generation; + DataT data; + }; + vector<Slot> slots_; + vector<int32> empty_slots_; + + Id encode_id(int32 id) const { + return (static_cast<uint64>(id) << 32) | slots_[id].generation; + } + + int32 decode_id(Id id) const { + int32 slot_id = static_cast<int32>(id >> 32); + uint32 generation = static_cast<uint32>(id); + if (slot_id < 0 || slot_id >= static_cast<int32>(slots_.size())) { + return -1; + } + if (generation != slots_[slot_id].generation) { + return -1; + } + return slot_id; + } + + int32 store(DataT &&data, uint8 type) { + int32 pos; + if (!empty_slots_.empty()) { + pos = empty_slots_.back(); + empty_slots_.pop_back(); + slots_[pos].data = std::move(data); + slots_[pos].generation ^= (slots_[pos].generation & TYPE_MASK) ^ type; + } else { + CHECK(slots_.size() <= static_cast<size_t>(std::numeric_limits<int32>::max())); + pos = static_cast<int32>(slots_.size()); + slots_.push_back(Slot{GENERATION_STEP + type, std::move(data)}); + } + return pos; + } + + void release(int32 id) { + inc_generation(id); + slots_[id].data = DataT(); + if (slots_[id].generation & ~TYPE_MASK) { // generation overflow. Can't use this id anymore + empty_slots_.push_back(id); + } + } + + void inc_generation(int32 id) { + slots_[id].generation += GENERATION_STEP; + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Enumerator.h b/libs/tdlib/td/tdutils/td/utils/Enumerator.h new file mode 100644 index 0000000000..ca7c0493ff --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Enumerator.h @@ -0,0 +1,45 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +#include <map> +#include <tuple> + +namespace td { + +template <class ValueT> +class Enumerator { + public: + using Key = int32; + + Key add(ValueT v) { + int32 next_id = narrow_cast<int32>(arr_.size() + 1); + bool was_inserted; + decltype(map_.begin()) it; + std::tie(it, was_inserted) = map_.emplace(std::move(v), next_id); + if (was_inserted) { + arr_.push_back(&it->first); + } + return it->second; + } + + const ValueT &get(Key key) const { + auto pos = static_cast<size_t>(key - 1); + CHECK(pos < arr_.size()); + return *arr_[pos]; + } + + private: + std::map<ValueT, int32> map_; + std::vector<const ValueT *> arr_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/FileLog.cpp b/libs/tdlib/td/tdutils/td/utils/FileLog.cpp new file mode 100644 index 0000000000..e3c84f1713 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/FileLog.cpp @@ -0,0 +1,92 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/FileLog.h" + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/port/path.h" +#include "td/utils/Slice.h" + +#include <limits> + +namespace td { + +bool FileLog::init(string path, int64 rotate_threshold) { + if (path == path_) { + set_rotate_threshold(rotate_threshold); + return true; + } + + auto r_fd = FileFd::open(path, FileFd::Create | FileFd::Write | FileFd::Append); + if (r_fd.is_error()) { + LOG(ERROR) << "Can't open log: " << r_fd.error(); + return false; + } + + fd_.close(); + fd_ = r_fd.move_as_ok(); + Fd::duplicate(fd_.get_fd(), Fd::Stderr()).ignore(); + + path_ = std::move(path); + size_ = fd_.get_size(); + rotate_threshold_ = rotate_threshold; + return true; +} + +void FileLog::set_rotate_threshold(int64 rotate_threshold) { + rotate_threshold_ = rotate_threshold; +} + +void FileLog::append(CSlice cslice, int log_level) { + Slice slice = cslice; + while (!slice.empty()) { + auto r_size = fd_.write(slice); + if (r_size.is_error()) { + process_fatal_error(r_size.error().message()); + } + auto written = r_size.ok(); + size_ += static_cast<int64>(written); + slice.remove_prefix(written); + } + if (log_level == VERBOSITY_NAME(FATAL)) { + process_fatal_error(cslice); + } + + if (size_ > rotate_threshold_) { + auto status = rename(path_, path_ + ".old"); + if (status.is_error()) { + process_fatal_error(status.message()); + } + do_rotate(); + } +} + +void FileLog::rotate() { + if (path_.empty()) { + return; + } + do_rotate(); +} + +void FileLog::do_rotate() { + auto current_verbosity_level = GET_VERBOSITY_LEVEL(); + SET_VERBOSITY_LEVEL(std::numeric_limits<int>::min()); // to ensure that nothing will be printed to the closed log + CHECK(!path_.empty()); + fd_.close(); + auto r_fd = FileFd::open(path_, FileFd::Create | FileFd::Truncate | FileFd::Write); + if (r_fd.is_error()) { + process_fatal_error(r_fd.error().message()); + } + fd_ = r_fd.move_as_ok(); + Fd::duplicate(fd_.get_fd(), Fd::Stderr()).ignore(); + size_ = 0; + SET_VERBOSITY_LEVEL(current_verbosity_level); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/FileLog.h b/libs/tdlib/td/tdutils/td/utils/FileLog.h new file mode 100644 index 0000000000..12e9d1479a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/FileLog.h @@ -0,0 +1,37 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" + +namespace td { + +class FileLog : public LogInterface { + static constexpr int64 DEFAULT_ROTATE_THRESHOLD = 10 * (1 << 20); + + public: + bool init(string path, int64 rotate_threshold = DEFAULT_ROTATE_THRESHOLD); + + void set_rotate_threshold(int64 rotate_threshold); + + void append(CSlice cslice, int log_level) override; + + void rotate() override; + + private: + FileFd fd_; + string path_; + int64 size_; + int64 rotate_threshold_; + + void do_rotate(); +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/FloodControlFast.h b/libs/tdlib/td/tdutils/td/utils/FloodControlFast.h new file mode 100644 index 0000000000..9f047881aa --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/FloodControlFast.h @@ -0,0 +1,62 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/TimedStat.h" + +namespace td { + +class FloodControlFast { + public: + uint32 add_event(int32 now) { + for (auto &limit : limits_) { + limit.stat_.add_event(CounterStat::Event(), now); + if (limit.stat_.get_stat(now).count_ > limit.count_) { + wakeup_at_ = max(wakeup_at_, now + limit.duration_ * 2); + } + } + return wakeup_at_; + } + uint32 get_wakeup_at() { + return wakeup_at_; + } + + void add_limit(uint32 duration, int32 count) { + limits_.push_back({TimedStat<CounterStat>(duration, 0), duration, count}); + } + + void clear_events() { + for (auto &limit : limits_) { + limit.stat_.clear_events(); + } + wakeup_at_ = 0; + } + + private: + class CounterStat { + public: + struct Event {}; + int32 count_ = 0; + void on_event(Event e) { + count_++; + } + void clear() { + count_ = 0; + } + }; + + uint32 wakeup_at_ = 0; + struct Limit { + TimedStat<CounterStat> stat_; + uint32 duration_; + int32 count_; + }; + std::vector<Limit> limits_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/FloodControlStrict.h b/libs/tdlib/td/tdutils/td/utils/FloodControlStrict.h new file mode 100644 index 0000000000..521fbbedc0 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/FloodControlStrict.h @@ -0,0 +1,97 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <limits> + +namespace td { + +// More strict implementaions of flood control than FloodControlFast. +// Should be just fine for small counters. +class FloodControlStrict { + public: + int32 add_event(int32 now) { + events_.push_back(Event{now}); + if (without_update_ > 0) { + without_update_--; + } else { + update(now); + } + return wakeup_at_; + } + + // no more than count in each duration. + void add_limit(int32 duration, int32 count) { + limits_.push_back(Limit{duration, count, 0}); + } + + int32 get_wakeup_at() { + return wakeup_at_; + } + + void clear_events() { + events_.clear(); + for (auto &limit : limits_) { + limit.pos_ = 0; + } + without_update_ = 0; + wakeup_at_ = 0; + } + + int32 update(int32 now) { + size_t min_pos = events_.size(); + + without_update_ = std::numeric_limits<size_t>::max(); + for (auto &limit : limits_) { + if (limit.pos_ + limit.count_ < events_.size()) { + limit.pos_ = events_.size() - limit.count_; + } + + // binary-search? :D + while (limit.pos_ < events_.size() && events_[limit.pos_].timestamp_ + limit.duration_ < now) { + limit.pos_++; + } + + if (limit.count_ + limit.pos_ <= events_.size()) { + CHECK(limit.count_ + limit.pos_ == events_.size()); + wakeup_at_ = max(wakeup_at_, events_[limit.pos_].timestamp_ + limit.duration_); + without_update_ = 0; + } else { + without_update_ = min(without_update_, limit.count_ + limit.pos_ - events_.size()); + } + + min_pos = min(min_pos, limit.pos_); + } + + if (min_pos * 2 > events_.size()) { + for (auto &limit : limits_) { + limit.pos_ -= min_pos; + } + events_.erase(events_.begin(), events_.begin() + min_pos); + } + return wakeup_at_; + } + + private: + int32 wakeup_at_ = 0; + struct Event { + int32 timestamp_; + }; + struct Limit { + int32 duration_; + int32 count_; + size_t pos_; + }; + size_t without_update_ = 0; + std::vector<Event> events_; + std::vector<Limit> limits_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/GitInfo.cpp b/libs/tdlib/td/tdutils/td/utils/GitInfo.cpp new file mode 100644 index 0000000000..976286b923 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/GitInfo.cpp @@ -0,0 +1,20 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/GitInfo.h" + +#include "auto/git_info.h" + +namespace td { + +CSlice GitInfo::commit() { + return GIT_COMMIT; +} +bool GitInfo::is_dirty() { + return GIT_DIRTY; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/GitInfo.h b/libs/tdlib/td/tdutils/td/utils/GitInfo.h new file mode 100644 index 0000000000..a3ba32602f --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/GitInfo.h @@ -0,0 +1,19 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Slice.h" + +namespace td { + +class GitInfo { + public: + static CSlice commit(); + static bool is_dirty(); +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Gzip.cpp b/libs/tdlib/td/tdutils/td/utils/Gzip.cpp new file mode 100644 index 0000000000..d4e60d6e29 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Gzip.cpp @@ -0,0 +1,191 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Gzip.h" + +char disable_linker_warning_about_empty_file_gzip_cpp TD_UNUSED; + +#if TD_HAVE_ZLIB +#include "td/utils/logging.h" + +#include <cstring> +#include <limits> + +#include <zlib.h> + +namespace td { + +class Gzip::Impl { + public: + z_stream stream_; + + // z_stream is not copyable nor movable + Impl() = default; + Impl(const Impl &other) = delete; + Impl &operator=(const Impl &other) = delete; + Impl(Impl &&other) = delete; + Impl &operator=(Impl &&other) = delete; + ~Impl() = default; +}; + +Status Gzip::init_encode() { + CHECK(mode_ == Empty); + init_common(); + mode_ = Encode; + int ret = deflateInit2(&impl_->stream_, 6, Z_DEFLATED, 15, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + return Status::Error("zlib deflate init failed"); + } + return Status::OK(); +} + +Status Gzip::init_decode() { + CHECK(mode_ == Empty); + init_common(); + mode_ = Decode; + int ret = inflateInit2(&impl_->stream_, MAX_WBITS + 32); + if (ret != Z_OK) { + return Status::Error("zlib inflate init failed"); + } + return Status::OK(); +} + +void Gzip::set_input(Slice input) { + CHECK(input_size_ == 0); + CHECK(!close_input_flag_); + CHECK(input.size() <= std::numeric_limits<uInt>::max()); + CHECK(impl_->stream_.avail_in == 0); + input_size_ = input.size(); + impl_->stream_.avail_in = static_cast<uInt>(input.size()); + impl_->stream_.next_in = reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); +} + +void Gzip::set_output(MutableSlice output) { + CHECK(output_size_ == 0); + CHECK(output.size() <= std::numeric_limits<uInt>::max()); + CHECK(impl_->stream_.avail_out == 0); + output_size_ = output.size(); + impl_->stream_.avail_out = static_cast<uInt>(output.size()); + impl_->stream_.next_out = reinterpret_cast<Bytef *>(output.data()); +} + +Result<Gzip::State> Gzip::run() { + while (true) { + int ret; + if (mode_ == Decode) { + ret = inflate(&impl_->stream_, Z_NO_FLUSH); + } else { + ret = deflate(&impl_->stream_, close_input_flag_ ? Z_FINISH : Z_NO_FLUSH); + } + + if (ret == Z_OK) { + return Running; + } + if (ret == Z_STREAM_END) { + // TODO(now): fail if input is not empty; + clear(); + return Done; + } + clear(); + return Status::Error(PSLICE() << "zlib error " << ret); + } +} + +size_t Gzip::left_input() const { + return impl_->stream_.avail_in; +} +size_t Gzip::left_output() const { + return impl_->stream_.avail_out; +} + +void Gzip::init_common() { + std::memset(&impl_->stream_, 0, sizeof(impl_->stream_)); + impl_->stream_.zalloc = Z_NULL; + impl_->stream_.zfree = Z_NULL; + impl_->stream_.opaque = Z_NULL; + impl_->stream_.avail_in = 0; + impl_->stream_.next_in = nullptr; + impl_->stream_.avail_out = 0; + impl_->stream_.next_out = nullptr; + + input_size_ = 0; + output_size_ = 0; + + close_input_flag_ = false; +} + +void Gzip::clear() { + if (mode_ == Decode) { + inflateEnd(&impl_->stream_); + } else if (mode_ == Encode) { + deflateEnd(&impl_->stream_); + } + mode_ = Empty; +} + +Gzip::Gzip() : impl_(make_unique<Impl>()) { +} + +Gzip::Gzip(Gzip &&other) = default; + +Gzip &Gzip::operator=(Gzip &&other) = default; + +Gzip::~Gzip() { + clear(); +} + +BufferSlice gzdecode(Slice s) { + Gzip gzip; + gzip.init_decode().ensure(); + auto message = ChainBufferWriter::create_empty(); + gzip.set_input(s); + gzip.close_input(); + double k = 2; + gzip.set_output(message.prepare_append(static_cast<size_t>(static_cast<double>(s.size()) * k))); + while (true) { + auto r_state = gzip.run(); + if (r_state.is_error()) { + return BufferSlice(); + } + auto state = r_state.ok(); + if (state == Gzip::Done) { + message.confirm_append(gzip.flush_output()); + break; + } + if (gzip.need_input()) { + return BufferSlice(); + } + if (gzip.need_output()) { + message.confirm_append(gzip.flush_output()); + k *= 1.5; + gzip.set_output(message.prepare_append(static_cast<size_t>(static_cast<double>(gzip.left_input()) * k))); + } + } + return message.extract_reader().move_as_buffer_slice(); +} + +BufferSlice gzencode(Slice s, double k) { + Gzip gzip; + gzip.init_encode().ensure(); + gzip.set_input(s); + gzip.close_input(); + size_t max_size = static_cast<size_t>(static_cast<double>(s.size()) * k); + BufferWriter message{max_size}; + gzip.set_output(message.prepare_append()); + auto r_state = gzip.run(); + if (r_state.is_error()) { + return BufferSlice(); + } + auto state = r_state.ok(); + if (state != Gzip::Done) { + return BufferSlice(); + } + message.confirm_append(gzip.flush_output()); + return message.as_buffer_slice(); +} + +} // namespace td +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/Gzip.h b/libs/tdlib/td/tdutils/td/utils/Gzip.h new file mode 100644 index 0000000000..dd5fba5bf5 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Gzip.h @@ -0,0 +1,104 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#if TD_HAVE_ZLIB +#include "td/utils/buffer.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +class Gzip { + public: + Gzip(); + Gzip(const Gzip &) = delete; + Gzip &operator=(const Gzip &) = delete; + Gzip(Gzip &&other); + Gzip &operator=(Gzip &&other); + ~Gzip(); + + enum Mode { Empty, Encode, Decode }; + Status init(Mode mode) TD_WARN_UNUSED_RESULT { + if (mode == Encode) { + return init_encode(); + } else if (mode == Decode) { + return init_decode(); + } + clear(); + return Status::OK(); + } + + Status init_encode() TD_WARN_UNUSED_RESULT; + + Status init_decode() TD_WARN_UNUSED_RESULT; + + void set_input(Slice input); + + void set_output(MutableSlice output); + + void close_input() { + close_input_flag_ = true; + } + + bool need_input() const { + return left_input() == 0; + } + + bool need_output() const { + return left_output() == 0; + } + + size_t left_input() const; + + size_t left_output() const; + + size_t used_input() const { + return input_size_ - left_input(); + } + + size_t used_output() const { + return output_size_ - left_output(); + } + + size_t flush_input() { + auto res = used_input(); + input_size_ = left_input(); + return res; + } + + size_t flush_output() { + auto res = used_output(); + output_size_ = left_output(); + return res; + } + + enum State { Running, Done }; + Result<State> run() TD_WARN_UNUSED_RESULT; + + private: + class Impl; + unique_ptr<Impl> impl_; + + size_t input_size_ = 0; + size_t output_size_ = 0; + bool close_input_flag_ = false; + Mode mode_ = Empty; + + void init_common(); + void clear(); +}; + +BufferSlice gzdecode(Slice s); + +BufferSlice gzencode(Slice s, double k = 0.9); + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp b/libs/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp new file mode 100644 index 0000000000..d225ef800e --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/GzipByteFlow.cpp @@ -0,0 +1,70 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/GzipByteFlow.h" + +char disable_linker_warning_about_empty_file_gzipbyteflow_cpp TD_UNUSED; + +#if TD_HAVE_ZLIB +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +namespace td { + +void GzipByteFlow::loop() { + while (true) { + if (gzip_.need_input()) { + auto slice = input_->prepare_read(); + if (slice.empty()) { + if (!is_input_active_) { + gzip_.close_input(); + } else { + break; + } + } else { + gzip_.set_input(input_->prepare_read()); + } + } + if (gzip_.need_output()) { + auto slice = output_.prepare_append(); + CHECK(!slice.empty()); + gzip_.set_output(slice); + } + auto r_state = gzip_.run(); + auto output_size = gzip_.flush_output(); + if (output_size) { + uncommited_size_ += output_size; + total_output_size_ += output_size; + if (total_output_size_ > max_output_size_) { + return finish(Status::Error("Max output size limit exceeded")); + } + output_.confirm_append(output_size); + } + + auto input_size = gzip_.flush_input(); + if (input_size) { + input_->confirm_read(input_size); + } + if (r_state.is_error()) { + return finish(r_state.move_as_error()); + } + auto state = r_state.ok(); + if (state == Gzip::Done) { + on_output_updated(); + return consume_input(); + } + } + if (uncommited_size_ >= MIN_UPDATE_SIZE) { + uncommited_size_ = 0; + on_output_updated(); + } +} + +constexpr size_t GzipByteFlow::MIN_UPDATE_SIZE; + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/GzipByteFlow.h b/libs/tdlib/td/tdutils/td/utils/GzipByteFlow.h new file mode 100644 index 0000000000..c7e07abd0a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/GzipByteFlow.h @@ -0,0 +1,48 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/ByteFlow.h" +#include "td/utils/Gzip.h" + +#include <limits> + +namespace td { + +#if TD_HAVE_ZLIB +class GzipByteFlow final : public ByteFlowBase { + public: + GzipByteFlow() = default; + + explicit GzipByteFlow(Gzip::Mode mode) { + gzip_.init(mode).ensure(); + } + + void init_decode() { + gzip_.init_decode().ensure(); + } + + void init_encode() { + gzip_.init_encode().ensure(); + } + + void set_max_output_size(size_t max_output_size) { + max_output_size_ = max_output_size; + } + + void loop() override; + + private: + Gzip gzip_; + size_t uncommited_size_ = 0; + size_t total_output_size_ = 0; + size_t max_output_size_ = std::numeric_limits<size_t>::max(); + static constexpr size_t MIN_UPDATE_SIZE = 1 << 14; +}; +#endif + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/HazardPointers.h b/libs/tdlib/td/tdutils/td/utils/HazardPointers.h new file mode 100644 index 0000000000..e13dc8022e --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/HazardPointers.h @@ -0,0 +1,133 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <array> +#include <atomic> + +namespace td { + +template <class T, int MaxPointersN = 1> +class HazardPointers { + public: + explicit HazardPointers(size_t threads_n) : threads_(threads_n) { + for (auto &data : threads_) { + for (auto &ptr : data.hazard) { + ptr = nullptr; + } + } + } + HazardPointers(const HazardPointers &other) = delete; + HazardPointers &operator=(const HazardPointers &other) = delete; + HazardPointers(HazardPointers &&other) = delete; + HazardPointers &operator=(HazardPointers &&other) = delete; + + class Holder { + public: + T *protect(std::atomic<T *> &to_protect) { + return do_protect(hazard_ptr_, to_protect); + } + Holder(const Holder &other) = delete; + Holder &operator=(const Holder &other) = delete; + Holder(Holder &&other) = default; // TODO + Holder &operator=(Holder &&other) = delete; + ~Holder() { + clear(); + } + void clear() { + hazard_ptr_.store(nullptr, std::memory_order_release); + } + + private: + friend class HazardPointers; + explicit Holder(std::atomic<T *> &ptr) : hazard_ptr_(ptr) { + } + std::atomic<T *> &hazard_ptr_; + }; + + Holder get_holder(size_t thread_id, size_t pos) { + return Holder(get_hazard_ptr(thread_id, pos)); + } + + void retire(size_t thread_id, T *ptr = nullptr) { + CHECK(thread_id < threads_.size()); + auto &data = threads_[thread_id]; + if (ptr) { + data.to_delete.push_back(std::unique_ptr<T>(ptr)); + } + for (auto it = data.to_delete.begin(); it != data.to_delete.end();) { + if (!is_protected(it->get())) { + it->reset(); + it = data.to_delete.erase(it); + } else { + ++it; + } + } + } + + // old inteface + T *protect(size_t thread_id, size_t pos, std::atomic<T *> &ptr) { + return do_protect(get_hazard_ptr(thread_id, pos), ptr); + } + void clear(size_t thread_id, size_t pos) { + do_clear(get_hazard_ptr(thread_id, pos)); + } + + size_t to_delete_size_unsafe() const { + size_t res = 0; + for (auto &thread : threads_) { + res += thread.to_delete.size(); + } + return res; + } + + private: + struct ThreadData { + std::array<std::atomic<T *>, MaxPointersN> hazard; + char pad[TD_CONCURRENCY_PAD - sizeof(std::array<std::atomic<T *>, MaxPointersN>)]; + + // stupid gc + std::vector<std::unique_ptr<T>> to_delete; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<std::unique_ptr<T>>)]; + }; + std::vector<ThreadData> threads_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::vector<ThreadData>)]; + + static T *do_protect(std::atomic<T *> &hazard_ptr, std::atomic<T *> &to_protect) { + T *saved = nullptr; + T *to_save; + while ((to_save = to_protect.load()) != saved) { + hazard_ptr.store(to_save); + saved = to_save; + } + return saved; + } + + static void do_clear(std::atomic<T *> &hazard_ptr) { + hazard_ptr.store(nullptr, std::memory_order_release); + } + + bool is_protected(T *ptr) { + for (auto &thread : threads_) { + for (auto &hazard_ptr : thread.hazard) { + if (hazard_ptr.load() == ptr) { + return true; + } + } + } + return false; + } + + std::atomic<T *> &get_hazard_ptr(size_t thread_id, size_t pos) { + return threads_[thread_id].hazard[pos]; + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Heap.h b/libs/tdlib/td/tdutils/td/utils/Heap.h new file mode 100644 index 0000000000..54ee391497 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Heap.h @@ -0,0 +1,152 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +namespace td { + +struct HeapNode { + bool in_heap() const { + return pos_ != -1; + } + bool is_top() const { + return pos_ == 0; + } + void remove() { + pos_ = -1; + } + int pos_ = -1; +}; + +template <class KeyT, int K = 4> +class KHeap { + public: + bool empty() const { + return array_.empty(); + } + size_t size() const { + return array_.size(); + } + + KeyT top_key() const { + return array_[0].key_; + } + + HeapNode *pop() { + CHECK(!empty()); + HeapNode *result = array_[0].node_; + result->remove(); + erase(0); + return result; + } + + void insert(KeyT key, HeapNode *node) { + CHECK(!node->in_heap()); + array_.push_back({key, node}); + fix_up(static_cast<int>(array_.size()) - 1); + } + + void fix(KeyT key, HeapNode *node) { + CHECK(node->in_heap()); + int pos = node->pos_; + KeyT old_key = array_[pos].key_; + array_[pos].key_ = key; + if (key < old_key) { + fix_up(pos); + } else { + fix_down(pos); + } + } + + void erase(HeapNode *node) { + CHECK(node->in_heap()); + int pos = node->pos_; + node->remove(); + erase(pos); + } + + template <class F> + void for_each(F &f) const { + for (auto &it : array_) { + f(it.key_, it.node_); + } + } + + void check() const { + for (size_t i = 0; i < array_.size(); i++) { + for (size_t j = i * K + 1; j < i * K + 1 + K && j < array_.size(); j++) { + CHECK(array_[i].key_ <= array_[j].key_) << i << " " << j; + } + } + } + + private: + struct Item { + KeyT key_; + HeapNode *node_; + }; + vector<Item> array_; + + void fix_up(int pos) { + auto item = array_[pos]; + + while (pos) { + int parent_pos = (pos - 1) / K; + auto parent_item = array_[parent_pos]; + + if (parent_item.key_ < item.key_) { + break; + } + + parent_item.node_->pos_ = pos; + array_[pos] = parent_item; + pos = parent_pos; + } + + item.node_->pos_ = pos; + array_[pos] = item; + } + + void fix_down(int pos) { + auto item = array_[pos]; + while (true) { + int left_pos = pos * K + 1; + int right_pos = min(left_pos + K, static_cast<int>(array_.size())); + int next_pos = pos; + KeyT next_key = item.key_; + for (int i = left_pos; i < right_pos; i++) { + KeyT i_key = array_[i].key_; + if (i_key < next_key) { + next_key = i_key; + next_pos = i; + } + } + if (next_pos == pos) { + break; + } + array_[pos] = array_[next_pos]; + array_[pos].node_->pos_ = pos; + pos = next_pos; + } + + item.node_->pos_ = pos; + array_[pos] = item; + } + + void erase(int pos) { + array_[pos] = array_.back(); + array_.pop_back(); + if (pos < static_cast<int>(array_.size())) { + fix_down(pos); + fix_up(pos); + } + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Hints.cpp b/libs/tdlib/td/tdutils/td/utils/Hints.cpp new file mode 100644 index 0000000000..1e7449a668 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Hints.cpp @@ -0,0 +1,191 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Hints.h" + +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/unicode.h" +#include "td/utils/utf8.h" + +#include <algorithm> + +namespace td { + +vector<string> Hints::get_words(Slice name) { + bool in_word = false; + string word; + vector<string> words; + auto pos = name.ubegin(); + auto end = name.uend(); + while (pos != end) { + uint32 code; + pos = next_utf8_unsafe(pos, &code); + + code = prepare_search_character(code); + if (code == 0) { + continue; + } + if (code == ' ') { + if (in_word) { + words.push_back(std::move(word)); + word.clear(); + in_word = false; + } + } else { + in_word = true; + append_utf8_character(word, code); + } + } + if (in_word) { + words.push_back(std::move(word)); + } + std::sort(words.begin(), words.end()); + + size_t new_words_size = 0; + for (size_t i = 0; i != words.size(); i++) { + if (i == words.size() - 1 || !begins_with(words[i + 1], words[i])) { + if (i != new_words_size) { + words[new_words_size] = std::move(words[i]); + } + // LOG(ERROR) << "Get word " << words[new_words_size]; + new_words_size++; + } + } + words.resize(new_words_size); + return words; +} + +void Hints::add(KeyT key, Slice name) { + // LOG(ERROR) << "Add " << key << ": " << name; + auto it = key_to_name_.find(key); + if (it != key_to_name_.end()) { + if (it->second == name) { + return; + } + auto old_words = get_words(it->second); + for (auto &old_word : old_words) { + vector<KeyT> &keys = word_to_keys_[old_word]; + auto key_it = std::find(keys.begin(), keys.end(), key); + CHECK(key_it != keys.end()); + if (keys.size() == 1) { + word_to_keys_.erase(old_word); + } else { + CHECK(keys.size() > 1); + *key_it = keys.back(); + keys.pop_back(); + } + } + } + if (name.empty()) { + if (it != key_to_name_.end()) { + key_to_name_.erase(it); + } + key_to_rating_.erase(key); + return; + } + auto words = get_words(name); + for (auto &word : words) { + vector<KeyT> &keys = word_to_keys_[word]; + CHECK(std::find(keys.begin(), keys.end(), key) == keys.end()); + keys.push_back(key); + } + key_to_name_[key] = name.str(); +} + +void Hints::set_rating(KeyT key, RatingT rating) { + // LOG(ERROR) << "Set rating " << key << ": " << rating; + key_to_rating_[key] = rating; +} + +vector<Hints::KeyT> Hints::search_word(const string &word) const { + // LOG(ERROR) << "Search word " << word; + vector<KeyT> results; + auto it = word_to_keys_.lower_bound(word); + while (it != word_to_keys_.end() && begins_with(it->first, word)) { + results.insert(results.end(), it->second.begin(), it->second.end()); + ++it; + } + + std::sort(results.begin(), results.end()); + results.erase(std::unique(results.begin(), results.end()), results.end()); + return results; +} + +std::pair<size_t, vector<Hints::KeyT>> Hints::search(Slice query, int32 limit, bool return_all_for_empty_query) const { + // LOG(ERROR) << "Search " << query; + vector<KeyT> results; + + if (limit < 0) { + return {key_to_name_.size(), std::move(results)}; + } + + auto words = get_words(query); + if (return_all_for_empty_query && words.empty()) { + results.reserve(key_to_name_.size()); + for (auto &it : key_to_name_) { + results.push_back(it.first); + } + } + + for (size_t i = 0; i < words.size(); i++) { + vector<KeyT> keys = search_word(words[i]); + if (i == 0) { + results = std::move(keys); + continue; + } + + // now need to intersect two lists + size_t results_pos = 0; + size_t keys_pos = 0; + size_t new_results_size = 0; + while (results_pos != results.size() && keys_pos != keys.size()) { + if (results[results_pos] < keys[keys_pos]) { + results_pos++; + } else if (results[results_pos] > keys[keys_pos]) { + keys_pos++; + } else { + results[new_results_size++] = results[results_pos]; + results_pos++; + keys_pos++; + } + } + results.resize(new_results_size); + } + + auto total_size = results.size(); + if (total_size < static_cast<size_t>(limit)) { + std::sort(results.begin(), results.end(), CompareByRating(key_to_rating_)); + } else { + std::partial_sort(results.begin(), results.begin() + limit, results.end(), CompareByRating(key_to_rating_)); + results.resize(limit); + } + + return {total_size, std::move(results)}; +} + +bool Hints::has_key(KeyT key) const { + return key_to_name_.find(key) != key_to_name_.end(); +} + +string Hints::key_to_string(KeyT key) const { + auto it = key_to_name_.find(key); + if (it == key_to_name_.end()) { + return string(); + } + return it->second; +} + +std::pair<size_t, vector<Hints::KeyT>> Hints::search_empty(int32 limit) const { + return search(Slice(), limit, true); +} + +size_t Hints::size() const { + return key_to_name_.size(); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Hints.h b/libs/tdlib/td/tdutils/td/utils/Hints.h new file mode 100644 index 0000000000..645896684a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Hints.h @@ -0,0 +1,76 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +#include <map> +#include <unordered_map> +#include <utility> + +namespace td { + +// TODO template KeyT +class Hints { + using KeyT = int64; + using RatingT = int64; + + public: + void add(KeyT key, Slice name); + + void remove(KeyT key) { + add(key, ""); + } + + void set_rating(KeyT key, RatingT rating); + + std::pair<size_t, vector<KeyT>> search( + Slice query, int32 limit, + bool return_all_for_empty_query = false) const; // TODO sort by name instead of sort by rating + + bool has_key(KeyT key) const; + + string key_to_string(KeyT key) const; + + std::pair<size_t, vector<KeyT>> search_empty(int32 limit) const; // == search("", limit, true) + + size_t size() const; + + private: + std::map<string, vector<KeyT>> word_to_keys_; + std::unordered_map<KeyT, string> key_to_name_; + std::unordered_map<KeyT, RatingT> key_to_rating_; + + static vector<string> get_words(Slice name); + + vector<KeyT> search_word(const string &word) const; + + class CompareByRating { + const std::unordered_map<KeyT, RatingT> &key_to_rating_; + + RatingT get_rating(const KeyT &key) const { + auto it = key_to_rating_.find(key); + if (it == key_to_rating_.end()) { + return RatingT(); + } + return it->second; + } + + public: + explicit CompareByRating(const std::unordered_map<KeyT, RatingT> &key_to_rating) : key_to_rating_(key_to_rating) { + } + + bool operator()(const KeyT &lhs, const KeyT &rhs) const { + auto lhs_rating = get_rating(lhs); + auto rhs_rating = get_rating(rhs); + return lhs_rating < rhs_rating || (lhs_rating == rhs_rating && lhs < rhs); + } + }; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/HttpUrl.cpp b/libs/tdlib/td/tdutils/td/utils/HttpUrl.cpp new file mode 100644 index 0000000000..55b66f7b3a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/HttpUrl.cpp @@ -0,0 +1,189 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/HttpUrl.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Parser.h" + +namespace td { + +string HttpUrl::get_url() const { + string result; + switch (protocol_) { + case Protocol::HTTP: + result += "http://"; + break; + case Protocol::HTTPS: + result += "https://"; + break; + default: + UNREACHABLE(); + } + if (!userinfo_.empty()) { + result += userinfo_; + result += '@'; + } + if (is_ipv6) { + result += '['; + } + result += host_; + if (is_ipv6) { + result += ']'; + } + if (specified_port_ > 0) { + result += ':'; + result += to_string(specified_port_); + } + CHECK(!query_.empty() && query_[0] == '/'); + result += query_; + return result; +} + +Result<HttpUrl> parse_url(MutableSlice url, HttpUrl::Protocol default_protocol) { + // url == [https?://][userinfo@]host[:port] + Parser parser(url); + string protocol_str = to_lower(parser.read_till_nofail(':')); + + HttpUrl::Protocol protocol; + if (parser.start_with("://")) { + parser.advance(3); + if (protocol_str == "http") { + protocol = HttpUrl::Protocol::HTTP; + } else if (protocol_str == "https") { + protocol = HttpUrl::Protocol::HTTPS; + } else { + return Status::Error("Unsupported URL protocol"); + } + } else { + parser = Parser(url); + protocol = default_protocol; + } + Slice userinfo_host_port = parser.read_till_nofail("/?#"); + + int port = 0; + const char *colon = userinfo_host_port.end() - 1; + while (colon > userinfo_host_port.begin() && *colon != ':' && *colon != ']' && *colon != '@') { + colon--; + } + Slice userinfo_host; + if (colon > userinfo_host_port.begin() && *colon == ':') { + port = to_integer<int>(Slice(colon + 1, userinfo_host_port.end())); + userinfo_host = Slice(userinfo_host_port.begin(), colon); + } else { + userinfo_host = userinfo_host_port; + } + if (port < 0 || port > 65535) { + return Status::Error("Wrong port number specified in the URL"); + } + + auto at_pos = userinfo_host.rfind('@'); + Slice userinfo = at_pos == static_cast<size_t>(-1) ? "" : userinfo_host.substr(0, at_pos); + Slice host = userinfo_host.substr(at_pos + 1); + + bool is_ipv6 = false; + if (!host.empty() && host[0] == '[' && host.back() == ']') { + host.remove_prefix(1); + host.remove_suffix(1); + is_ipv6 = true; + } + if (host.empty()) { + return Status::Error("URL host is empty"); + } + + int specified_port = port; + if (port == 0) { + if (protocol == HttpUrl::Protocol::HTTP) { + port = 80; + } else { + CHECK(protocol == HttpUrl::Protocol::HTTPS); + port = 443; + } + } + + Slice query = parser.read_all(); + while (!query.empty() && is_space(query.back())) { + query.remove_suffix(1); + } + if (query.empty()) { + query = "/"; + } + string query_str; + if (query[0] != '/') { + query_str = '/'; + } + for (auto c : query) { + if (static_cast<unsigned char>(c) <= 0x20) { + query_str += '%'; + query_str += "0123456789ABCDEF"[c / 16]; + query_str += "0123456789ABCDEF"[c % 16]; + } else { + query_str += c; + } + } + + string host_str = to_lower(host); + for (size_t i = 0; i < host_str.size(); i++) { + char c = host_str[i]; + if (('a' <= c && c <= 'z') || c == '.' || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '!' || c == '$' || + c == ',' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')' || c == ';' || c == '&' || c == '+' || + c == '=') { + // symbols allowed by RFC 7230 and RFC 3986 + continue; + } + if (c == '%') { + c = host_str[++i]; + if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) { + c = host_str[++i]; + if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) { + // percent encoded symbol as allowed by RFC 7230 and RFC 3986 + continue; + } + } + } + // all other symbols aren't allowed + unsigned char uc = static_cast<unsigned char>(c); + if (uc >= 128) { + // but we allow plain UTF-8 symbols + continue; + } + return Status::Error("Wrong URL host"); + } + + return HttpUrl{protocol, userinfo.str(), std::move(host_str), is_ipv6, specified_port, port, std::move(query_str)}; +} + +StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) { + sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::HTTP ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_) + << tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_); + return sb; +} + +string get_url_query_file_name(const string &query) { + Slice query_slice = query; + query_slice.truncate(query.find_first_of("?#")); + + auto slash_pos = query_slice.rfind('/'); + if (slash_pos < query_slice.size()) { + return query_slice.substr(slash_pos + 1).str(); + } + return query_slice.str(); +} + +string get_url_file_name(const string &url) { + // TODO remove copy + string url_copy = url; + auto r_http_url = parse_url(url_copy); + if (r_http_url.is_error()) { + LOG(WARNING) << "Receive wrong URL \"" << url << '"'; + return string(); + } + return get_url_query_file_name(r_http_url.ok().query_); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/HttpUrl.h b/libs/tdlib/td/tdutils/td/utils/HttpUrl.h new file mode 100644 index 0000000000..f7d1e4aaba --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/HttpUrl.h @@ -0,0 +1,39 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +namespace td { + +class HttpUrl { + public: + enum class Protocol { HTTP, HTTPS } protocol_; + string userinfo_; + string host_; + bool is_ipv6; + int specified_port_; + int port_; + string query_; + + string get_url() const; +}; + +// TODO Slice instead of MutableSlice +Result<HttpUrl> parse_url(MutableSlice url, + HttpUrl::Protocol default_protocol = HttpUrl::Protocol::HTTP) TD_WARN_UNUSED_RESULT; + +StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url); + +string get_url_query_file_name(const string &query); + +string get_url_file_name(const string &url); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/JsonBuilder.cpp b/libs/tdlib/td/tdutils/td/utils/JsonBuilder.cpp new file mode 100644 index 0000000000..eb654f43cd --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/JsonBuilder.cpp @@ -0,0 +1,648 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/JsonBuilder.h" + +#include "td/utils/misc.h" +#include "td/utils/ScopeGuard.h" + +#include <cstring> + +namespace td { +StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val) { + sb << '"'; + SCOPE_EXIT { + sb << '"'; + }; + auto *s = val.value_.begin(); + auto len = val.value_.size(); + + for (size_t pos = 0; pos < len; pos++) { + auto ch = static_cast<unsigned char>(s[pos]); + switch (ch) { + case '"': + sb << '\\' << '"'; + break; + case '\\': + sb << '\\' << '\\'; + break; + case '\b': + sb << '\\' << 'b'; + break; + case '\f': + sb << '\\' << 'f'; + break; + case '\n': + sb << '\\' << 'n'; + break; + case '\r': + sb << '\\' << 'r'; + break; + case '\t': + sb << '\\' << 't'; + break; + default: + if (ch <= 31) { + sb << JsonOneChar(s[pos]); + break; + } + sb << s[pos]; + break; + } + } + return sb; +} + +StringBuilder &operator<<(StringBuilder &sb, const JsonString &val) { + sb << '"'; + SCOPE_EXIT { + sb << '"'; + }; + auto *s = val.str_.begin(); + auto len = val.str_.size(); + + for (size_t pos = 0; pos < len; pos++) { + auto ch = static_cast<unsigned char>(s[pos]); + switch (ch) { + case '"': + sb << '\\' << '"'; + break; + case '\\': + sb << '\\' << '\\'; + break; + case '\b': + sb << '\\' << 'b'; + break; + case '\f': + sb << '\\' << 'f'; + break; + case '\n': + sb << '\\' << 'n'; + break; + case '\r': + sb << '\\' << 'r'; + break; + case '\t': + sb << '\\' << 't'; + break; + default: + if (ch <= 31) { + sb << JsonOneChar(s[pos]); + break; + } + if (128 <= ch) { + int a = s[pos]; + CHECK((a & 0x40) != 0); + + CHECK(pos + 1 < len); + int b = s[++pos]; + CHECK((b & 0xc0) == 0x80); + if ((a & 0x20) == 0) { + CHECK((a & 0x1e) > 0); + sb << JsonChar(((a & 0x1f) << 6) | (b & 0x3f)); + break; + } + + CHECK(pos + 1 < len); + int c = s[++pos]; + CHECK((c & 0xc0) == 0x80); + if ((a & 0x10) == 0) { + CHECK(((a & 0x0f) | (b & 0x20)) > 0); + sb << JsonChar(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + break; + } + + CHECK(pos + 1 < len); + int d = s[++pos]; + CHECK((d & 0xc0) == 0x80); + if ((a & 0x08) == 0) { + CHECK(((a & 0x07) | (b & 0x30)) > 0); + sb << JsonChar(((a & 0x07) << 18) | ((b & 0x3f) << 12) | ((c & 0x3f) << 6) | (d & 0x3f)); + break; + } + + UNREACHABLE(); + break; + } + sb << s[pos]; + break; + } + } + return sb; +} +Result<MutableSlice> json_string_decode(Parser &parser) { + if (!parser.try_skip('"')) { + return Status::Error("Opening '\"' expected"); + } + auto *cur_src = parser.data().data(); + auto *end_src = parser.data().end(); + auto *end = cur_src; + while (end < end_src && end[0] != '"') { + if (end[0] == '\\') { + end++; + } + end++; + } + if (end >= end_src) { + return Status::Error("Closing '\"' not found"); + } + parser.advance(end + 1 - cur_src); + end_src = end; + + auto *cur_dest = cur_src; + auto *begin_dest = cur_src; + + while (cur_src != end_src) { + auto *slash = static_cast<char *>(std::memchr(cur_src, '\\', end_src - cur_src)); + if (slash == nullptr) { + slash = end_src; + } + std::memmove(cur_dest, cur_src, slash - cur_src); + cur_dest += slash - cur_src; + cur_src = slash; + if (cur_src != end_src) { + cur_src++; + if (cur_src == end_src) { + // TODO UNREACHABLE(); + return Status::Error("Unexpected end of string"); + } + switch (*cur_src) { + case '"': + case '\\': + case '/': + *cur_dest++ = *cur_src++; + break; + case 'b': + *cur_dest++ = '\b'; + cur_src++; + break; + case 'f': + *cur_dest++ = '\f'; + cur_src++; + break; + case 'n': + *cur_dest++ = '\n'; + cur_src++; + break; + case 'r': + *cur_dest++ = '\r'; + cur_src++; + break; + case 't': + *cur_dest++ = '\t'; + cur_src++; + break; + case 'u': { + cur_src++; + if (cur_src + 4 > end_src) { + return Status::Error("\\u has less than 4 symbols"); + } + int num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + num = num * 16 + d; + } + if (0xD7FF < num && num < 0xE000) { + if (cur_src + 6 <= end_src && cur_src[0] == '\\' && cur_src[1] == 'u') { + cur_src += 2; + int new_num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + new_num = new_num * 16 + d; + } + if (0xD7FF < new_num && new_num < 0xE000) { + num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000; + } else { + cur_src -= 6; + } + } + } + + if (num < 128) { + *cur_dest++ = static_cast<char>(num); + } else if (num < 0x800) { + *cur_dest++ = static_cast<char>(0xc0 + (num >> 6)); + *cur_dest++ = static_cast<char>(0x80 + (num & 63)); + } else if (num < 0xffff) { + *cur_dest++ = static_cast<char>(0xe0 + (num >> 12)); + *cur_dest++ = static_cast<char>(0x80 + ((num >> 6) & 63)); + *cur_dest++ = static_cast<char>(0x80 + (num & 63)); + } else { + *cur_dest++ = static_cast<char>(0xf0 + (num >> 18)); + *cur_dest++ = static_cast<char>(0x80 + ((num >> 12) & 63)); + *cur_dest++ = static_cast<char>(0x80 + ((num >> 6) & 63)); + *cur_dest++ = static_cast<char>(0x80 + (num & 63)); + } + break; + } + } + } + } + CHECK(cur_dest <= end_src); + return MutableSlice(begin_dest, cur_dest); +} + +Status json_string_skip(Parser &parser) { + if (!parser.try_skip('"')) { + return Status::Error("Opening '\"' expected"); + } + auto *begin_src = parser.data().data(); + auto *cur_src = begin_src; + auto *end_src = parser.data().end(); + auto *end = cur_src; + while (end < end_src && *end != '"') { + if (*end == '\\') { + end++; + } + end++; + } + if (end >= end_src) { + return Status::Error("Closing '\"' not found"); + } + parser.advance(end + 1 - cur_src); + end_src = end; + + while (cur_src != end_src) { + auto *slash = static_cast<char *>(std::memchr(cur_src, '\\', end_src - cur_src)); + if (slash == nullptr) { + slash = end_src; + } + cur_src = slash; + if (cur_src != end_src) { + cur_src++; + if (cur_src == end_src) { + // TODO UNREACHABLE(); + return Status::Error("Unexpected end of string"); + } + switch (*cur_src) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + cur_src++; + break; + case 'u': { + cur_src++; + if (cur_src + 4 > end_src) { + return Status::Error("\\u has less than 4 symbols"); + } + int num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + num = num * 16 + d; + } + if (0xD7FF < num && num < 0xE000) { + if (cur_src + 6 <= end_src && cur_src[0] == '\\' && cur_src[1] == 'u') { + cur_src += 2; + int new_num = 0; + for (int i = 0; i < 4; i++, cur_src++) { + int d = hex_to_int(*cur_src); + if (d == 16) { + return Status::Error("Invalid \\u -- not hex digit"); + } + new_num = new_num * 16 + d; + } + if (0xD7FF < new_num && new_num < 0xE000) { + // num = (((num & 0x3FF) << 10) | (new_num & 0x3FF)) + 0x10000; + } else { + cur_src -= 6; + } + } + } + break; + } + } + } + } + return Status::OK(); +} + +Result<JsonValue> do_json_decode(Parser &parser, int32 max_depth) { + if (max_depth < 0) { + return Status::Error("Too big object depth"); + } + + parser.skip_whitespaces(); + switch (parser.peek_char()) { + case 'f': + if (parser.skip_start_with("false")) { + return JsonValue::create_boolean(false); + } + return Status::Error("Starts with 'f' -- false expected"); + case 't': + if (parser.skip_start_with("true")) { + return JsonValue::create_boolean(true); + } + return Status::Error("Starts with 't' -- true expected"); + case 'n': + if (parser.skip_start_with("null")) { + return JsonValue(); + } + return Status::Error("Starts with 'n' -- null expected"); + case '"': { + TRY_RESULT(slice, json_string_decode(parser)); + return JsonValue::create_string(slice); + } + case '[': { + parser.skip('['); + parser.skip_whitespaces(); + std::vector<JsonValue> res; + if (parser.try_skip(']')) { + return JsonValue::create_array(std::move(res)); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_RESULT(value, do_json_decode(parser, max_depth - 1)); + res.emplace_back(std::move(value)); + + parser.skip_whitespaces(); + if (parser.try_skip(']')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return JsonValue::create_array(std::move(res)); + } + case '{': { + parser.skip('{'); + parser.skip_whitespaces(); + std::vector<std::pair<MutableSlice, JsonValue> > res; + if (parser.try_skip('}')) { + return JsonValue::make_object(std::move(res)); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_RESULT(key, json_string_decode(parser)); + parser.skip_whitespaces(); + if (!parser.try_skip(':')) { + return Status::Error("':' expected"); + } + TRY_RESULT(value, do_json_decode(parser, max_depth - 1)); + res.emplace_back(std::move(key), std::move(value)); + + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return JsonValue::make_object(std::move(res)); + } + case '-': + case '+': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + auto num = parser.read_while( + [](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; }); + return JsonValue::create_number(num); + } + case 0: + return Status::Error("Unexpected end"); + default: { + char next = parser.peek_char(); + if (0 < next && next < 127) { + return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'"); + } else { + return Status::Error("Unexpected symbol"); + } + } + } + UNREACHABLE(); +} + +Status do_json_skip(Parser &parser, int32 max_depth) { + if (max_depth < 0) { + return Status::Error("Too big object depth"); + } + + parser.skip_whitespaces(); + switch (parser.peek_char()) { + case 'f': + if (parser.skip_start_with("false")) { + return Status::OK(); + } + return Status::Error("Starts with 'f' -- false expected"); + case 't': + if (parser.skip_start_with("true")) { + return Status::OK(); + } + return Status::Error("Starts with 't' -- true expected"); + case 'n': + if (parser.skip_start_with("null")) { + return Status::OK(); + } + return Status::Error("Starts with 'n' -- null expected"); + case '"': { + return json_string_skip(parser); + } + case '[': { + parser.skip('['); + parser.skip_whitespaces(); + if (parser.try_skip(']')) { + return Status::OK(); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_STATUS(do_json_skip(parser, max_depth - 1)); + + parser.skip_whitespaces(); + if (parser.try_skip(']')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return Status::OK(); + } + case '{': { + parser.skip('{'); + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + return Status::OK(); + } + while (true) { + if (parser.empty()) { + return Status::Error("Unexpected end"); + } + TRY_STATUS(json_string_skip(parser)); + parser.skip_whitespaces(); + if (!parser.try_skip(':')) { + return Status::Error("':' expected"); + } + TRY_STATUS(do_json_skip(parser, max_depth - 1)); + + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + break; + } + if (parser.try_skip(',')) { + parser.skip_whitespaces(); + continue; + } + return Status::Error("Unexpected symbol"); + } + return Status::OK(); + } + case '-': + case '+': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + parser.read_while( + [](char c) { return c == '-' || ('0' <= c && c <= '9') || c == 'e' || c == 'E' || c == '+' || c == '.'; }); + return Status::OK(); + } + case 0: + return Status::Error("Unexpected end"); + default: { + char next = parser.peek_char(); + if (0 < next && next < 127) { + return Status::Error(PSLICE() << "Unexpected symbol '" << parser.peek_char() << "'"); + } else { + return Status::Error("Unexpected symbol"); + } + } + } + return Status::Error("Can't parse"); +} + +Slice JsonValue::get_type_name(Type type) { + switch (type) { + case Type::Null: + return Slice("Null"); + case Type::Number: + return Slice("Number"); + case Type::Boolean: + return Slice("Boolean"); + case Type::String: + return Slice("String"); + case Type::Array: + return Slice("Array"); + case Type::Object: + return Slice("Object"); + default: + UNREACHABLE(); + return Slice("Unknown"); + } +} + +bool has_json_object_field(JsonObject &object, Slice name) { + for (auto &field_value : object) { + if (field_value.first == name) { + return true; + } + } + return false; +} + +Result<JsonValue> get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, bool is_optional) { + for (auto &field_value : object) { + if (field_value.first == name) { + if (type != JsonValue::Type::Null && field_value.second.type() != type) { + return Status::Error(400, PSLICE() + << "Field \"" << name << "\" must be of type " << JsonValue::get_type_name(type)); + } + + return std::move(field_value.second); + } + } + if (!is_optional) { + return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\""); + } + return JsonValue(); +} + +Result<bool> get_json_object_bool_field(JsonObject &object, Slice name, bool is_optional, bool default_value) { + TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Boolean, is_optional)); + if (value.type() == JsonValue::Type::Null) { + return default_value; + } + return value.get_boolean(); +} + +Result<int32> get_json_object_int_field(JsonObject &object, Slice name, bool is_optional, int32 default_value) { + TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Number, is_optional)); + if (value.type() == JsonValue::Type::Null) { + return default_value; + } + return to_integer_safe<int32>(value.get_number()); +} + +Result<double> get_json_object_double_field(JsonObject &object, Slice name, bool is_optional, double default_value) { + TRY_RESULT(value, get_json_object_field(object, name, JsonValue::Type::Number, is_optional)); + if (value.type() == JsonValue::Type::Null) { + return default_value; + } + return to_double(value.get_number()); +} + +Result<string> get_json_object_string_field(JsonObject &object, Slice name, bool is_optional, string default_value) { + for (auto &field_value : object) { + if (field_value.first == name) { + if (field_value.second.type() == JsonValue::Type::String) { + return field_value.second.get_string().str(); + } + if (field_value.second.type() == JsonValue::Type::Number) { + return field_value.second.get_number().str(); + } + + return Status::Error(400, PSLICE() << "Field \"" << name << "\" must be of type String"); + } + } + if (is_optional) { + return default_value; + } + return Status::Error(400, PSLICE() << "Can't find field \"" << name << "\""); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/JsonBuilder.h b/libs/tdlib/td/tdutils/td/utils/JsonBuilder.h new file mode 100644 index 0000000000..735c4b29ec --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/JsonBuilder.h @@ -0,0 +1,760 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Parser.h" +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include <new> +#include <tuple> +#include <type_traits> +#include <utility> + +namespace td { + +template <class... Args> +std::tuple<const Args &...> ctie(const Args &... args) TD_WARN_UNUSED_RESULT; + +template <class... Args> +std::tuple<const Args &...> ctie(const Args &... args) { + return std::tie(args...); +} + +class JsonTrue { + public: + friend StringBuilder &operator<<(StringBuilder &sb, const JsonTrue &val) { + return sb << "true"; + } +}; + +class JsonFalse { + public: + friend StringBuilder &operator<<(StringBuilder &sb, const JsonFalse &val) { + return sb << "false"; + } +}; + +class JsonNull { + public: + friend StringBuilder &operator<<(StringBuilder &sb, JsonNull val) { + return sb << "null"; + } +}; + +class JsonBool { + public: + explicit JsonBool(bool value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonBool &val) { + if (val.value_) { + return sb << JsonTrue(); + } else { + return sb << JsonFalse(); + } + } + + private: + bool value_; +}; + +class JsonInt { + public: + explicit JsonInt(int32 value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonInt &val) { + return sb << val.value_; + } + + private: + int32 value_; +}; + +class JsonLong { + public: + explicit JsonLong(int64 value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonLong &val) { + return sb << val.value_; + } + + private: + int64 value_; +}; + +class JsonFloat { + public: + explicit JsonFloat(double value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonFloat &val) { + return sb << val.value_; + } + + private: + double value_; +}; + +class JsonOneChar { + public: + explicit JsonOneChar(unsigned int c) : c_(c) { + } + + friend StringBuilder &operator<<(StringBuilder &sb, const JsonOneChar &val) { + auto c = val.c_; + return sb << '\\' << 'u' << "0123456789abcdef"[c >> 12] << "0123456789abcdef"[(c >> 8) & 15] + << "0123456789abcdef"[(c >> 4) & 15] << "0123456789abcdef"[c & 15]; + } + + private: + unsigned int c_; +}; + +class JsonChar { + public: + explicit JsonChar(unsigned int c) : c_(c) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonChar &val) { + auto c = val.c_; + if (c < 0x10000) { + if (0xD7FF < c && c < 0xE000) { + // UTF-8 correctness has already been checked + UNREACHABLE(); + } + return sb << JsonOneChar(c); + } else if (c <= 0x10ffff) { + return sb << JsonOneChar(0xD7C0 + (c >> 10)) << JsonOneChar(0xDC00 + (c & 0x3FF)); + } else { + // UTF-8 correctness has already been checked + UNREACHABLE(); + } + } + + private: + unsigned int c_; +}; + +class JsonRaw { + public: + explicit JsonRaw(Slice value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonRaw &val) { + return sb << val.value_; + } + + private: + Slice value_; +}; + +class JsonRawString { + public: + explicit JsonRawString(Slice value) : value_(value) { + } + friend StringBuilder &operator<<(StringBuilder &sb, const JsonRawString &val); + + private: + Slice value_; +}; + +class JsonString { + public: + explicit JsonString(Slice str) : str_(str) { + } + + friend StringBuilder &operator<<(StringBuilder &sb, const JsonString &val); + + private: + Slice str_; +}; + +class JsonScope; +class JsonValueScope; +class JsonArrayScope; +class JsonObjectScope; + +class JsonBuilder { + public: + explicit JsonBuilder(StringBuilder &&sb) : sb_(std::move(sb)) { + } + StringBuilder &string_builder() { + return sb_; + } + friend class JsonScope; + JsonValueScope enter_value() TD_WARN_UNUSED_RESULT; + JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT; + JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT; + + private: + StringBuilder sb_; + JsonScope *scope_ = nullptr; +}; + +class Jsonable {}; + +class JsonScope { + public: + explicit JsonScope(JsonBuilder *jb) : sb_(&jb->sb_), jb_(jb) { + save_scope_ = jb_->scope_; + jb_->scope_ = this; + CHECK(is_active()); + } + JsonScope(const JsonScope &other) = delete; + JsonScope(JsonScope &&other) : sb_(other.sb_), jb_(other.jb_), save_scope_(other.save_scope_) { + other.jb_ = nullptr; + } + JsonScope &operator=(const JsonScope &) = delete; + JsonScope &operator=(JsonScope &&) = delete; + ~JsonScope() { + if (jb_) { + leave(); + } + } + void leave() { + CHECK(is_active()); + jb_->scope_ = save_scope_; + } + + protected: + StringBuilder *sb_; + + // For CHECK + JsonBuilder *jb_; + JsonScope *save_scope_; + + bool is_active() const { + return jb_ && jb_->scope_ == this; + } + + JsonScope &operator<<(JsonTrue x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(JsonFalse x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(JsonNull x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonBool &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonInt &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonLong &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonFloat &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonString &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonRawString &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(const JsonRaw &x) { + *sb_ << x; + return *this; + } + JsonScope &operator<<(bool x) { + return *this << JsonBool(x); + } + JsonScope &operator<<(int32 x) { + return *this << JsonInt(x); + } + JsonScope &operator<<(int64 x) { + return *this << JsonLong(x); + } + JsonScope &operator<<(double x) { + return *this << JsonFloat(x); + } + template <class T> + JsonScope &operator<<(const T *x); // not implemented + template <size_t N> + JsonScope &operator<<(const char (&x)[N]) { + return *this << JsonString(Slice(x)); + } + JsonScope &operator<<(const char *x) { + return *this << JsonString(Slice(x)); + } + JsonScope &operator<<(const string &x) { + return *this << JsonString(Slice(x)); + } + JsonScope &operator<<(Slice x) { + return *this << JsonString(x); + } +}; + +class JsonValueScope : public JsonScope { + public: + using JsonScope::JsonScope; + template <class T> + std::enable_if_t<std::is_base_of<Jsonable, typename std::decay<T>::type>::value, JsonValueScope &> operator<<( + const T &x) { + x.store(this); + return *this; + } + template <class T> + std::enable_if_t<!std::is_base_of<Jsonable, typename std::decay<T>::type>::value, JsonValueScope &> operator<<( + const T &x) { + CHECK(!was_); + was_ = true; + JsonScope::operator<<(x); + return *this; + } + + JsonArrayScope enter_array() TD_WARN_UNUSED_RESULT; + JsonObjectScope enter_object() TD_WARN_UNUSED_RESULT; + + private: + bool was_ = false; +}; + +class JsonArrayScope : public JsonScope { + public: + explicit JsonArrayScope(JsonBuilder *jb) : JsonScope(jb) { + *sb_ << "["; + } + JsonArrayScope(JsonArrayScope &&other) = default; + ~JsonArrayScope() { + if (jb_) { + leave(); + } + } + void leave() { + *sb_ << "]"; + } + template <class T> + JsonArrayScope &operator<<(const T &x) { + enter_value() << x; + return *this; + } + JsonValueScope enter_value() { + CHECK(is_active()); + if (is_first_) { + *sb_ << ","; + } else { + is_first_ = true; + } + return jb_->enter_value(); + } + + private: + bool is_first_ = false; +}; + +class JsonObjectScope : public JsonScope { + public: + explicit JsonObjectScope(JsonBuilder *jb) : JsonScope(jb) { + *sb_ << "{"; + } + JsonObjectScope(JsonObjectScope &&other) = default; + ~JsonObjectScope() { + if (jb_) { + leave(); + } + } + void leave() { + *sb_ << "}"; + } + template <class S, class T> + JsonObjectScope &operator<<(std::tuple<S, T> key_value) { + return *this << std::pair<S, T>(std::get<0>(key_value), std::get<1>(key_value)); + } + template <class S, class T> + JsonObjectScope &operator<<(std::pair<S, T> key_value) { + CHECK(is_active()); + if (is_first_) { + *sb_ << ","; + } else { + is_first_ = true; + } + jb_->enter_value() << key_value.first; + *sb_ << ":"; + jb_->enter_value() << key_value.second; + return *this; + } + JsonObjectScope &operator<<(const JsonRaw &key_value) { + CHECK(is_active()); + is_first_ = true; + jb_->enter_value() << key_value; + return *this; + } + + private: + bool is_first_ = false; +}; + +inline JsonArrayScope JsonValueScope::enter_array() { + CHECK(!was_); + was_ = true; + return JsonArrayScope(jb_); +} +inline JsonObjectScope JsonValueScope::enter_object() { + CHECK(!was_); + was_ = true; + return JsonObjectScope(jb_); +} +inline JsonValueScope JsonBuilder::enter_value() { + return JsonValueScope(this); +} +inline JsonObjectScope JsonBuilder::enter_object() { + return JsonObjectScope(this); +} +inline JsonArrayScope JsonBuilder::enter_array() { + return JsonArrayScope(this); +} + +class JsonValue; + +using JsonObject = vector<std::pair<MutableSlice, JsonValue>>; +using JsonArray = vector<JsonValue>; + +class JsonValue : public Jsonable { + public: + enum class Type { Null, Number, Boolean, String, Array, Object }; + + static Slice get_type_name(Type type); + + JsonValue() { + } + ~JsonValue() { + destroy(); + } + JsonValue(JsonValue &&other) : JsonValue() { + init(std::move(other)); + } + JsonValue &operator=(JsonValue &&other) { + if (&other == this) { + return *this; + } + destroy(); + init(std::move(other)); + return *this; + } + JsonValue(const JsonValue &other) = delete; + JsonValue &operator=(const JsonValue &other) = delete; + + Type type() const { + return type_; + } + + MutableSlice &get_string() { + CHECK(type_ == Type::String); + return string_; + } + const MutableSlice &get_string() const { + CHECK(type_ == Type::String); + return string_; + } + bool &get_boolean() { + CHECK(type_ == Type::Boolean); + return boolean_; + } + const bool &get_boolean() const { + CHECK(type_ == Type::Boolean); + return boolean_; + } + + MutableSlice &get_number() { + CHECK(type_ == Type::Number); + return number_; + } + const MutableSlice &get_number() const { + CHECK(type_ == Type::Number); + return number_; + } + + JsonArray &get_array() { + CHECK(type_ == Type::Array); + return array_; + } + const JsonArray &get_array() const { + CHECK(type_ == Type::Array); + return array_; + } + + JsonObject &get_object() { + CHECK(type_ == Type::Object); + return object_; + } + const JsonObject &get_object() const { + CHECK(type_ == Type::Object); + return object_; + } + + static JsonValue create_boolean(bool val) { + JsonValue res; + res.init_boolean(val); + return res; + } + + static JsonValue create_number(MutableSlice number) { + JsonValue res; + res.init_number(number); + return res; + } + + static JsonValue create_string(MutableSlice str) { + JsonValue res; + res.init_string(str); + return res; + } + + static JsonValue create_array(JsonArray v) { + JsonValue res; + res.init_array(std::move(v)); + return res; + } + + static JsonValue make_object(JsonObject c) { + JsonValue res; + res.init_object(std::move(c)); + return res; + } + + void store(JsonValueScope *scope) const { + switch (type_) { + case Type::Null: + *scope << JsonRaw("null"); + break; + case Type::Boolean: + if (get_boolean()) { + *scope << JsonRaw("true"); + } else { + *scope << JsonRaw("false"); + } + break; + case Type::Number: + *scope << JsonRaw(get_number()); + break; + case Type::String: + *scope << JsonString(get_string()); + break; + case Type::Array: { + auto arr = scope->enter_array(); + for (auto &val : get_array()) { + arr << val; + } + break; + } + case Type::Object: { + auto object = scope->enter_object(); + for (auto &key_value : get_object()) { + object << ctie(JsonString(key_value.first), key_value.second); + } + break; + } + } + }; + + private: + Type type_{Type::Null}; + union { + MutableSlice number_; + bool boolean_; + MutableSlice string_; + JsonArray array_; + JsonObject object_; + }; + + void init_null() { + type_ = Type::Null; + } + void init_number(MutableSlice number) { + type_ = Type::Number; + new (&number_) MutableSlice(number); + } + void init_boolean(bool boolean) { + type_ = Type::Boolean; + boolean_ = boolean; + } + void init_string(MutableSlice slice) { + type_ = Type::String; + new (&string_) MutableSlice(slice); + } + void init_array(JsonArray array) { + type_ = Type::Array; + new (&array_) JsonArray(std::move(array)); + } + void init_object(JsonObject object) { + type_ = Type::Object; + new (&object_) JsonObject(std::move(object)); + } + + void init(JsonValue &&other) { + switch (other.type_) { + case Type::Null: + break; + case Type::Number: + init_number(other.number_); + break; + case Type::Boolean: + init_boolean(other.boolean_); + break; + case Type::String: + init_string(other.string_); + break; + case Type::Array: + init_array(std::move(other.array_)); + break; + case Type::Object: + init_object(std::move(other.object_)); + break; + } + other.destroy(); + } + + void destroy() { + switch (type_) { + case Type::Null: + case Type::Boolean: + break; + case Type::Number: + number_.~MutableSlice(); + break; + case Type::String: + string_.~MutableSlice(); + break; + case Type::Array: + array_.~vector<JsonValue>(); + break; + case Type::Object: + object_.~vector<std::pair<MutableSlice, JsonValue>>(); + break; + } + type_ = Type::Null; + } +}; + +inline StringBuilder &operator<<(StringBuilder &sb, JsonValue::Type type) { + switch (type) { + case JsonValue::Type::Object: + return sb << "JsonObject"; + case JsonValue::Type::Boolean: + return sb << "JsonBoolean"; + case JsonValue::Type::Null: + return sb << "JsonNull"; + case JsonValue::Type::Number: + return sb << "JsonNumber"; + case JsonValue::Type::Array: + return sb << "JsonArray"; + case JsonValue::Type::String: + return sb << "JsonString"; + default: + UNREACHABLE(); + return sb; + } +} + +class VirtuallyJsonable : public Jsonable { + public: + virtual void store(JsonValueScope *scope) const = 0; + VirtuallyJsonable() = default; + VirtuallyJsonable(const VirtuallyJsonable &) = delete; + VirtuallyJsonable &operator=(const VirtuallyJsonable &) = delete; + VirtuallyJsonable(VirtuallyJsonable &&) = default; + VirtuallyJsonable &operator=(VirtuallyJsonable &&) = default; + virtual ~VirtuallyJsonable() = default; +}; + +class VirtuallyJsonableInt : public VirtuallyJsonable { + public: + explicit VirtuallyJsonableInt(int32 value) : value_(value) { + } + void store(JsonValueScope *scope) const override { + *scope << JsonInt(value_); + } + + private: + int32 value_; +}; + +class VirtuallyJsonableLong : public VirtuallyJsonable { + public: + explicit VirtuallyJsonableLong(int64 value) : value_(value) { + } + void store(JsonValueScope *scope) const override { + *scope << JsonLong(value_); + } + + private: + int64 value_; +}; + +class VirtuallyJsonableString : public VirtuallyJsonable { + public: + explicit VirtuallyJsonableString(Slice value) : value_(value) { + } + void store(JsonValueScope *scope) const override { + *scope << JsonString(value_); + } + + private: + Slice value_; +}; + +Result<MutableSlice> json_string_decode(Parser &parser) TD_WARN_UNUSED_RESULT; +Status json_string_skip(Parser &parser) TD_WARN_UNUSED_RESULT; + +Result<JsonValue> do_json_decode(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT; +Status do_json_skip(Parser &parser, int32 max_depth) TD_WARN_UNUSED_RESULT; + +inline Result<JsonValue> json_decode(MutableSlice from) { + Parser parser(from); + const int32 DEFAULT_MAX_DEPTH = 100; + auto result = do_json_decode(parser, DEFAULT_MAX_DEPTH); + if (result.is_ok()) { + parser.skip_whitespaces(); + if (!parser.empty()) { + return Status::Error("Expected string end"); + } + } + return result; +} + +template <class StrT, class ValT> +StrT json_encode(const ValT &val) { + auto buf_len = 1 << 19; + auto buf = StackAllocator::alloc(buf_len); + JsonBuilder jb(StringBuilder(buf.as_slice())); + jb.enter_value() << val; + LOG_IF(ERROR, jb.string_builder().is_error()) << "Json buffer overflow"; + auto slice = jb.string_builder().as_cslice(); + return StrT(slice.begin(), slice.size()); +} + +bool has_json_object_field(JsonObject &object, Slice name); + +Result<JsonValue> get_json_object_field(JsonObject &object, Slice name, JsonValue::Type type, + bool is_optional = true) TD_WARN_UNUSED_RESULT; + +Result<bool> get_json_object_bool_field(JsonObject &object, Slice name, bool is_optional = true, + bool default_value = false) TD_WARN_UNUSED_RESULT; + +Result<int32> get_json_object_int_field(JsonObject &object, Slice name, bool is_optional = true, + int32 default_value = 0) TD_WARN_UNUSED_RESULT; + +Result<double> get_json_object_double_field(JsonObject &object, Slice name, bool is_optional = true, + double default_value = 0.0) TD_WARN_UNUSED_RESULT; + +Result<string> get_json_object_string_field(JsonObject &object, Slice name, bool is_optional = true, + string default_value = "") TD_WARN_UNUSED_RESULT; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/List.h b/libs/tdlib/td/tdutils/td/utils/List.h new file mode 100644 index 0000000000..1606c44d2b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/List.h @@ -0,0 +1,92 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/logging.h" + +namespace td { + +struct ListNode { + ListNode *next; + ListNode *prev; + ListNode() { + clear(); + } + + ~ListNode() { + remove(); + } + + ListNode(const ListNode &) = delete; + ListNode &operator=(const ListNode &) = delete; + + ListNode(ListNode &&other) { + if (other.empty()) { + clear(); + } else { + ListNode *head = other.prev; + other.remove(); + head->put(this); + } + } + + ListNode &operator=(ListNode &&other) { + this->remove(); + + if (!other.empty()) { + ListNode *head = other.prev; + other.remove(); + head->put(this); + } + + return *this; + } + + void connect(ListNode *to) { + CHECK(to != nullptr); + next = to; + to->prev = this; + } + + void remove() { + prev->connect(next); + clear(); + } + + void put(ListNode *other) { + other->connect(next); + this->connect(other); + } + + void put_back(ListNode *other) { + prev->connect(other); + other->connect(this); + } + + ListNode *get() { + ListNode *result = prev; + if (result == this) { + return nullptr; + } + result->prev->connect(this); + result->clear(); + // this->connect(result->next); + return result; + } + + bool empty() const { + return next == this; + } + + private: + void clear() { + next = this; + prev = this; + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MemoryLog.h b/libs/tdlib/td/tdutils/td/utils/MemoryLog.h new file mode 100644 index 0000000000..aa125df2f7 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MemoryLog.h @@ -0,0 +1,83 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +#include <atomic> +#include <cstdio> +#include <cstring> + +namespace td { + +template <int buffer_size = 32 * (1 << 10)> +class MemoryLog : public LogInterface { + static constexpr size_t MAX_OUTPUT_SIZE = buffer_size / 16 < (8 << 10) ? buffer_size / 16 : (8 << 10); + + public: + MemoryLog() { + std::memset(buffer_, ' ', sizeof(buffer_)); + } + + void append(CSlice new_slice, int log_level) override { + Slice slice = new_slice; + slice.truncate(MAX_OUTPUT_SIZE); + while (!slice.empty() && slice.back() == '\n') { + slice.remove_suffix(1); + } + size_t slice_size = slice.size(); + CHECK(slice_size * 3 < buffer_size); + size_t pad_size = ((slice_size + 15) & ~15) - slice_size; + constexpr size_t magic_size = 16; + uint32 total_size = static_cast<uint32>(slice_size + pad_size + magic_size); + uint32 real_pos = pos_.fetch_add(total_size, std::memory_order_relaxed); + CHECK((total_size & 15) == 0); + + uint32 start_pos = real_pos & (buffer_size - 1); + uint32 end_pos = start_pos + total_size; + if (likely(end_pos <= buffer_size)) { + std::memcpy(&buffer_[start_pos + magic_size], slice.data(), slice_size); + std::memcpy(&buffer_[start_pos + magic_size + slice_size], " ", pad_size); + } else { + size_t first = buffer_size - start_pos - magic_size; + size_t second = slice_size - first; + std::memcpy(&buffer_[start_pos + magic_size], slice.data(), first); + std::memcpy(&buffer_[0], slice.data() + first, second); + std::memcpy(&buffer_[second], " ", pad_size); + } + + CHECK((start_pos & 15) == 0); + CHECK(start_pos <= buffer_size - magic_size); + buffer_[start_pos] = '\n'; + size_t printed = std::snprintf(&buffer_[start_pos + 1], magic_size - 1, "LOG:%08x: ", real_pos); + CHECK(printed == magic_size - 2); + buffer_[start_pos + magic_size - 1] = ' '; + + if (log_level == VERBOSITY_NAME(FATAL)) { + process_fatal_error(new_slice); + } + } + + void rotate() override { + } + + Slice get_buffer() const { + return Slice(buffer_, sizeof(buffer_)); + } + + size_t get_pos() const { + return pos_ & (buffer_size - 1); + } + + private: + char buffer_[buffer_size]; + std::atomic<uint32> pos_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MimeType.cpp b/libs/tdlib/td/tdutils/td/utils/MimeType.cpp new file mode 100644 index 0000000000..75c4fe34b5 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MimeType.cpp @@ -0,0 +1,44 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/MimeType.h" + +#include "td/utils/logging.h" + +const char *extension_to_mime_type(const char *extension, size_t extension_len); // auto-generated +const char *mime_type_to_extension(const char *mime_type, size_t mime_type_len); // auto-generated + +namespace td { + +string MimeType::to_extension(Slice mime_type, Slice default_value) { + if (mime_type.empty()) { + return default_value.str(); + } + + const char *result = ::mime_type_to_extension(mime_type.data(), mime_type.size()); + if (result != nullptr) { + return result; + } + + LOG(INFO) << "Unknown file MIME type " << mime_type; + return default_value.str(); +} + +string MimeType::from_extension(Slice extension, Slice default_value) { + if (extension.empty()) { + return default_value.str(); + } + + const char *result = ::extension_to_mime_type(extension.data(), extension.size()); + if (result != nullptr) { + return result; + } + + LOG(INFO) << "Unknown file extension " << extension; + return default_value.str(); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MimeType.h b/libs/tdlib/td/tdutils/td/utils/MimeType.h new file mode 100644 index 0000000000..11210ceb30 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MimeType.h @@ -0,0 +1,20 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +class MimeType { + public: + static string to_extension(Slice mime_type, Slice default_value = Slice()); + static string from_extension(Slice extension, Slice default_value = Slice()); +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MovableValue.h b/libs/tdlib/td/tdutils/td/utils/MovableValue.h new file mode 100644 index 0000000000..939bf51f28 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MovableValue.h @@ -0,0 +1,40 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +namespace td { + +template <class T, T empty_val = T()> +class MovableValue { + public: + MovableValue() = default; + MovableValue(T val) : val_(val) { + } + MovableValue(MovableValue &&other) : val_(other.val_) { + other.clear(); + } + MovableValue &operator=(MovableValue &&other) { + val_ = other.val_; + other.clear(); + return *this; + } + MovableValue(const MovableValue &) = delete; + MovableValue &operator=(const MovableValue &) = delete; + ~MovableValue() = default; + + void clear() { + val_ = empty_val; + } + const T &get() const { + return val_; + } + + private: + T val_ = empty_val; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MpmcQueue.h b/libs/tdlib/td/tdutils/td/utils/MpmcQueue.h new file mode 100644 index 0000000000..ae65554b72 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MpmcQueue.h @@ -0,0 +1,449 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +// MPMC queue +// Simple semaphore protected implementation +// To close queue, one should send as much sentinel elements as there are readers. +// Once there are no readers and writers, one may easily destroy queue + +#include "td/utils/format.h" +#include "td/utils/HazardPointers.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" +#include "td/utils/ScopeGuard.h" + +#include <array> +#include <atomic> + +namespace td { + +namespace detail { +struct MpmcStat { + void alloc_ok(size_t thread_id) { + s(thread_id).alloc_ok_cnt++; + } + void alloc_error(size_t thread_id) { + s(thread_id).alloc_error_cnt++; + } + void push_loop_error(size_t thread_id) { + s(thread_id).push_loop_error_cnt++; + } + void push_loop_ok(size_t thread_id) { + s(thread_id).push_loop_ok_cnt++; + } + void dump() { + int alloc_ok_cnt = 0; + int alloc_error_cnt = 0; + int push_loop_error_cnt = 0; + int push_loop_ok_cnt = 0; + for (auto &d : arr) { + alloc_ok_cnt += d.alloc_ok_cnt; + alloc_error_cnt += d.alloc_error_cnt; + push_loop_error_cnt += d.push_loop_error_cnt; + push_loop_ok_cnt += d.push_loop_ok_cnt; + } + LOG(ERROR) << tag("alloc_ok_cnt", alloc_ok_cnt) << tag("alloc_error_cnt", alloc_error_cnt) + << tag("push_loop_error_cnt", push_loop_error_cnt) << tag("push_loop_ok_cnt", push_loop_ok_cnt); + } + + private: + struct ThreadStat { + int alloc_ok_cnt{0}; + int alloc_error_cnt{0}; + int push_loop_ok_cnt{0}; + int push_loop_error_cnt{0}; + char pad[TD_CONCURRENCY_PAD - sizeof(int) * 4]; + }; + std::array<ThreadStat, 1024> arr; + ThreadStat &s(size_t thread_id) { + return arr[thread_id]; + } +}; +} // namespace detail +//detail::MpmcStat stat_; + +template <class T> +class OneValue { + public: + bool set_value(T &value) { + value_ = std::move(value); + int state = Empty; + if (state_.compare_exchange_strong(state, Value, std::memory_order_acq_rel)) { + return true; + } + value = std::move(value_); + return false; + } + bool get_value(T &value) { + auto old_state = state_.exchange(Taken, std::memory_order_acq_rel); + if (old_state == Value) { + value = std::move(value_); + return true; + } + return false; + } + void reset() { + state_ = Empty; + value_ = T(); + } + + private: + enum Type : int { Empty = 0, Taken, Value }; + std::atomic<int> state_{Empty}; + T value_; +}; + +template <class T> +class OneValue<T *> { + public: + bool set_value(T *value) { + T *was = nullptr; + return state_.compare_exchange_strong(was, value, std::memory_order_acq_rel); + } + bool get_value(T *&value) { + value = state_.exchange(Taken(), std::memory_order_acq_rel); + return value != nullptr; + } + void reset() { + state_ = nullptr; + } + OneValue() { + } + + private: + std::atomic<T *> state_{nullptr}; + T *Taken() { + static T xxx; + return &xxx; + } +}; + +template <class T> +class MpmcQueueBlock { + public: + explicit MpmcQueueBlock(size_t size) : nodes_(size) { + } + enum class PopStatus { Ok, Empty, Closed }; + + //blocking pop + //returns Ok or Closed + PopStatus pop(T &value) { + while (true) { + auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed); + if (read_pos >= nodes_.size()) { + return PopStatus::Closed; + } + //TODO blocking get_value + if (nodes_[static_cast<size_t>(read_pos)].one_value.get_value(value)) { + return PopStatus::Ok; + } + } + } + + //nonblocking pop + //returns Ok, Empty or Closed + PopStatus try_pop(T &value) { + while (true) { + auto read_pos = read_pos_.fetch_add(1, std::memory_order_relaxed); + if (read_pos >= nodes_.size()) { + return PopStatus::Closed; + } + if (nodes_[static_cast<size_t>(read_pos)].one_value.get_value(value)) { + return PopStatus::Ok; + } + auto write_pos = write_pos_.load(std::memory_order_relaxed); + if (write_pos <= read_pos + 1) { + return PopStatus::Empty; + } + } + } + + enum class PushStatus { Ok, Closed }; + PushStatus push(T &value) { + while (true) { + auto write_pos = write_pos_.fetch_add(1, std::memory_order_relaxed); + if (write_pos >= nodes_.size()) { + return PushStatus::Closed; + } + if (nodes_[static_cast<size_t>(write_pos)].one_value.set_value(value)) { + //stat_.push_loop_ok(0); + return PushStatus::Ok; + } + //stat_.push_loop_error(0); + } + } + + private: + struct Node { + OneValue<T> one_value; + }; + std::atomic<uint64> write_pos_{0}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)]; + std::atomic<uint64> read_pos_{0}; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)]; + std::vector<Node> nodes_; + char pad3[TD_CONCURRENCY_PAD - sizeof(std::vector<Node>)]; +}; + +template <class T> +class MpmcQueueOld { + public: + explicit MpmcQueueOld(size_t threads_n) : MpmcQueueOld(1024, threads_n) { + } + static std::string get_description() { + return "Mpmc queue (fetch and add array queue)"; + } + MpmcQueueOld(size_t block_size, size_t threads_n) : block_size_{block_size}, hazard_pointers_{threads_n} { + auto node = std::make_unique<Node>(block_size_); + write_pos_ = node.get(); + read_pos_ = node.get(); + node.release(); + } + + MpmcQueueOld(const MpmcQueueOld &other) = delete; + MpmcQueueOld &operator=(const MpmcQueueOld &other) = delete; + MpmcQueueOld(MpmcQueueOld &&other) = delete; + MpmcQueueOld &operator=(MpmcQueueOld &&other) = delete; + ~MpmcQueueOld() { + auto *ptr = read_pos_.load(std::memory_order_relaxed); + while (ptr) { + auto *to_delete = ptr; + ptr = ptr->next_.load(std::memory_order_relaxed); + delete to_delete; + } + //stat_.dump(); + //stat_ = MpmcStat(); + } + + size_t hazard_pointers_to_delele_size_unsafe() const { + return hazard_pointers_.to_delete_size_unsafe(); + } + void gc(size_t thread_id) { + hazard_pointers_.retire(thread_id); + } + + using PushStatus = typename MpmcQueueBlock<T>::PushStatus; + using PopStatus = typename MpmcQueueBlock<T>::PopStatus; + + void push(T value, size_t thread_id) { + auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0); + while (true) { + auto node = hazard_ptr_holder.protect(write_pos_); + auto status = node->block.push(value); + switch (status) { + case PushStatus::Ok: + return; + case PushStatus::Closed: { + auto next = node->next_.load(std::memory_order_acquire); + if (next == nullptr) { + auto new_node = new Node(block_size_); + new_node->block.push(value); + if (node->next_.compare_exchange_strong(next, new_node, std::memory_order_acq_rel)) { + //stat_.alloc_ok(thread_id); + write_pos_.compare_exchange_strong(node, new_node, std::memory_order_acq_rel); + return; + } else { + //stat_.alloc_error(thread_id); + new_node->block.pop(value); + //CHECK(status == PopStatus::Ok); + delete new_node; + } + } + //CHECK(next != nullptr); + write_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel); + break; + } + } + } + } + + bool try_pop(T &value, size_t thread_id) { + auto hazard_ptr_holder = hazard_pointers_.get_holder(thread_id, 0); + while (true) { + auto node = hazard_ptr_holder.protect(read_pos_); + auto status = node->block.try_pop(value); + switch (status) { + case PopStatus::Ok: + return true; + case PopStatus::Empty: + return false; + case PopStatus::Closed: { + auto next = node->next_.load(std::memory_order_acquire); + if (!next) { + return false; + } + if (read_pos_.compare_exchange_strong(node, next, std::memory_order_acq_rel)) { + hazard_ptr_holder.clear(); + hazard_pointers_.retire(thread_id, node); + } + break; + } + } + } + } + + T pop(size_t thread_id) { + T value; + while (true) { + if (try_pop(value, thread_id)) { + return value; + } + td::this_thread::yield(); + } + } + + private: + struct Node { + explicit Node(size_t block_size) : block{block_size} { + } + std::atomic<Node *> next_{nullptr}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)]; + MpmcQueueBlock<T> block; + //Got pad in MpmcQueueBlock + }; + std::atomic<Node *> write_pos_; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)]; + std::atomic<Node *> read_pos_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)]; + size_t block_size_; + HazardPointers<Node, 1> hazard_pointers_; + //Got pad in HazardPointers +}; + +template <class T> +class MpmcQueue { + public: + explicit MpmcQueue(size_t threads_n) : MpmcQueue(1024, threads_n) { + } + static std::string get_description() { + return "NEW Mpmc queue (fetch and add array queue)"; + } + MpmcQueue(size_t block_size, size_t threads_n) : hazard_pointers_{threads_n} { + auto node = std::make_unique<Node>(); + write_pos_ = node.get(); + read_pos_ = node.get(); + node.release(); + } + + MpmcQueue(const MpmcQueue &other) = delete; + MpmcQueue &operator=(const MpmcQueue &other) = delete; + MpmcQueue(MpmcQueue &&other) = delete; + MpmcQueue &operator=(MpmcQueue &&other) = delete; + ~MpmcQueue() { + auto *ptr = read_pos_.load(std::memory_order_relaxed); + while (ptr) { + auto *to_delete = ptr; + ptr = ptr->next.load(std::memory_order_relaxed); + delete to_delete; + } + } + + size_t hazard_pointers_to_delele_size_unsafe() const { + return hazard_pointers_.to_delete_size_unsafe(); + } + void gc(size_t thread_id) { + hazard_pointers_.retire(thread_id); + } + + void push(T value, size_t thread_id) { + SCOPE_EXIT { + hazard_pointers_.clear(thread_id, 0); + }; + while (true) { + auto node = hazard_pointers_.protect(thread_id, 0, write_pos_); + auto &block = node->block; + auto pos = block.write_pos++; + if (pos >= block.data.size()) { + auto next = node->next.load(); + if (next == nullptr) { + auto new_node = new Node{}; + new_node->block.write_pos++; + new_node->block.data[0].set_value(value); + Node *null = nullptr; + if (node->next.compare_exchange_strong(null, new_node)) { + write_pos_.compare_exchange_strong(node, new_node); + return; + } else { + new_node->block.data[0].get_value(value); + delete new_node; + } + } else { + write_pos_.compare_exchange_strong(node, next); + } + } else { + if (block.data[static_cast<size_t>(pos)].set_value(value)) { + return; + } + } + } + } + + bool try_pop(T &value, size_t thread_id) { + SCOPE_EXIT { + hazard_pointers_.clear(thread_id, 0); + }; + while (true) { + auto node = hazard_pointers_.protect(thread_id, 0, read_pos_); + auto &block = node->block; + if (block.write_pos <= block.read_pos && node->next.load(std::memory_order_relaxed) == nullptr) { + return false; + } + auto pos = block.read_pos++; + if (pos >= block.data.size()) { + auto next = node->next.load(); + if (!next) { + return false; + } + if (read_pos_.compare_exchange_strong(node, next)) { + hazard_pointers_.clear(thread_id, 0); + hazard_pointers_.retire(thread_id, node); + } + } else { + if (block.data[static_cast<size_t>(pos)].get_value(value)) { + return true; + } + } + } + } + + T pop(size_t thread_id) { + T value; + while (true) { + if (try_pop(value, thread_id)) { + return value; + } + td::this_thread::yield(); + } + } + + private: + struct Block { + std::atomic<uint64> write_pos{0}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)]; + std::atomic<uint64> read_pos{0}; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<uint64>)]; + std::array<OneValue<T>, 1024> data; + char pad3[TD_CONCURRENCY_PAD]; + }; + struct Node { + Node() = default; + + Block block; + std::atomic<Node *> next{nullptr}; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)]; + //Got pad in MpmcQueueBlock + }; + std::atomic<Node *> write_pos_; + char pad[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)]; + std::atomic<Node *> read_pos_; + char pad2[TD_CONCURRENCY_PAD - sizeof(std::atomic<Node *>)]; + HazardPointers<Node, 1> hazard_pointers_; + //Got pad in HazardPointers +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MpmcWaiter.h b/libs/tdlib/td/tdutils/td/utils/MpmcWaiter.h new file mode 100644 index 0000000000..0f48620e63 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MpmcWaiter.h @@ -0,0 +1,106 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/thread.h" + +#include <atomic> +#include <condition_variable> +#include <mutex> + +namespace td { + +class MpmcWaiter { + public: + int wait(int yields, uint32 worker_id) { + if (yields < RoundsTillSleepy) { + td::this_thread::yield(); + return yields + 1; + } else if (yields == RoundsTillSleepy) { + auto state = state_.load(std::memory_order_relaxed); + if (!State::has_worker(state)) { + auto new_state = State::with_worker(state, worker_id); + if (state_.compare_exchange_strong(state, new_state)) { + td::this_thread::yield(); + return yields + 1; + } + if (state == State::awake()) { + return 0; + } + } + td::this_thread::yield(); + return 0; + } else if (yields < RoundsTillAsleep) { + auto state = state_.load(std::memory_order_acquire); + if (State::still_sleepy(state, worker_id)) { + td::this_thread::yield(); + return yields + 1; + } + return 0; + } else { + auto state = state_.load(std::memory_order_acquire); + if (State::still_sleepy(state, worker_id)) { + std::unique_lock<std::mutex> lock(mutex_); + if (state_.compare_exchange_strong(state, State::asleep(), std::memory_order_acq_rel)) { + condition_variable_.wait(lock); + } + } + return 0; + } + } + + int stop_wait(int yields, uint32 worker_id) { + if (yields > RoundsTillSleepy) { + notify_cold(); + } + return 0; + } + + void notify() { + if (state_.load(std::memory_order_acquire) == State::awake()) { + return; + } + notify_cold(); + } + + private: + struct State { + static constexpr uint32 awake() { + return 0; + } + static constexpr uint32 asleep() { + return 1; + } + static bool is_asleep(uint32 state) { + return (state & 1) != 0; + } + static bool has_worker(uint32 state) { + return (state >> 1) != 0; + } + static int32 with_worker(uint32 state, uint32 worker) { + return state | ((worker + 1) << 1); + } + static bool still_sleepy(uint32 state, uint32 worker) { + return (state >> 1) == (worker + 1); + } + }; + enum { RoundsTillSleepy = 32, RoundsTillAsleep = 64 }; + std::atomic<uint32> state_{State::awake()}; + std::mutex mutex_; + std::condition_variable condition_variable_; + + void notify_cold() { + auto old_state = state_.exchange(State::awake(), std::memory_order_release); + if (State::is_asleep(old_state)) { + std::lock_guard<std::mutex> guard(mutex_); + condition_variable_.notify_all(); + } + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MpscLinkQueue.h b/libs/tdlib/td/tdutils/td/utils/MpscLinkQueue.h new file mode 100644 index 0000000000..4398c7503d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MpscLinkQueue.h @@ -0,0 +1,174 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <atomic> + +namespace td { +//NB: holder of the queue holds all responsibility of freeing its nodes +class MpscLinkQueueImpl { + public: + class Node; + class Reader; + + void push(Node *node) { + node->next_ = head_.load(std::memory_order_relaxed); + while (!head_.compare_exchange_strong(node->next_, node, std::memory_order_release, std::memory_order_relaxed)) { + } + } + + void push_unsafe(Node *node) { + node->next_ = head_.load(std::memory_order_relaxed); + head_.store(node, std::memory_order_relaxed); + } + + void pop_all(Reader &reader) { + return reader.add(head_.exchange(nullptr, std::memory_order_acquire)); + } + + void pop_all_unsafe(Reader &reader) { + return reader.add(head_.exchange(nullptr, std::memory_order_relaxed)); + } + + class Node { + friend class MpscLinkQueueImpl; + Node *next_{nullptr}; + }; + + class Reader { + public: + Node *read() { + auto old_head = head_; + if (head_) { + head_ = head_->next_; + } + return old_head; + } + void delay(Node *node) { + node->next_ = head_; + if (!head_) { + tail_ = node; + } + head_ = node; + } + size_t calc_size() const { + size_t res = 0; + for (auto it = head_; it != nullptr; it = it->next_, res++) { + } + return res; + } + + private: + friend class MpscLinkQueueImpl; + void add(Node *node) { + if (node == nullptr) { + return; + } + // Reverse list + Node *tail = node; + Node *head = nullptr; + while (node) { + auto next = node->next_; + node->next_ = head; + head = node; + node = next; + } + if (head_ == nullptr) { + head_ = head; + } else { + tail_->next_ = head; + } + tail_ = tail; + } + Node *head_{nullptr}; + Node *tail_{nullptr}; + }; + + private: + std::atomic<Node *> head_{nullptr}; +}; + +// Uses MpscLinkQueueImpl. +// Node should have to_mpsc_link_queue_node and from_mpsc_link_queue_node functions +template <class Node> +class MpscLinkQueue { + public: + void push(Node node) { + impl_.push(node.to_mpsc_link_queue_node()); + } + void push_unsafe(Node node) { + impl_.push_unsafe(node.to_mpsc_link_queue_node()); + } + class Reader { + public: + ~Reader() { + CHECK(!read()); + } + Node read() { + auto node = impl_.read(); + if (!node) { + return {}; + } + return Node::from_mpsc_link_queue_node(node); + } + void delay(Node node) { + impl_.delay(node.to_mpsc_link_queue_node()); + } + size_t calc_size() const { + return impl_.calc_size(); + } + + private: + friend class MpscLinkQueue; + + MpscLinkQueueImpl::Reader impl_; + MpscLinkQueueImpl::Reader &impl() { + return impl_; + } + }; + + void pop_all(Reader &reader) { + return impl_.pop_all(reader.impl()); + } + void pop_all_unsafe(Reader &reader) { + return impl_.pop_all_unsafe(reader.impl()); + } + + private: + MpscLinkQueueImpl impl_; +}; + +template <class Value> +class MpscLinkQueueUniquePtrNode { + public: + MpscLinkQueueUniquePtrNode() = default; + explicit MpscLinkQueueUniquePtrNode(std::unique_ptr<Value> ptr) : ptr_(std::move(ptr)) { + } + + MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() { + return ptr_.release()->to_mpsc_link_queue_node(); + } + static MpscLinkQueueUniquePtrNode<Value> from_mpsc_link_queue_node(td::MpscLinkQueueImpl::Node *node) { + return MpscLinkQueueUniquePtrNode<Value>(std::unique_ptr<Value>(Value::from_mpsc_link_queue_node(node))); + } + + explicit operator bool() { + return ptr_ != nullptr; + } + + Value &value() { + return *ptr_; + } + + private: + std::unique_ptr<Value> ptr_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/MpscPollableQueue.h b/libs/tdlib/td/tdutils/td/utils/MpscPollableQueue.h new file mode 100644 index 0000000000..89d2df8693 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/MpscPollableQueue.h @@ -0,0 +1,154 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/misc.h" +#include "td/utils/port/EventFd.h" +#include "td/utils/SpinLock.h" + +#if !TD_EVENTFD_UNSUPPORTED +#if !TD_WINDOWS +#include <poll.h> +#include <sched.h> +#endif + +#include <utility> + +namespace td { +// interface like in PollableQueue +template <class ValueT> +class MpscPollableQueue { + public: + int reader_wait_nonblock() { + auto ready = reader_vector_.size() - reader_pos_; + if (ready != 0) { + return narrow_cast<int>(ready); + } + + auto guard = lock_.lock(); + if (writer_vector_.empty()) { + event_fd_.acquire(); + wait_event_fd_ = true; + return 0; + } else { + reader_vector_.clear(); + reader_pos_ = 0; + std::swap(writer_vector_, reader_vector_); + return narrow_cast<int>(reader_vector_.size()); + } + } + ValueT reader_get_unsafe() { + return std::move(reader_vector_[reader_pos_++]); + } + void reader_flush() { + //nop + } + void writer_put(ValueT value) { + auto guard = lock_.lock(); + writer_vector_.push_back(std::move(value)); + if (wait_event_fd_) { + wait_event_fd_ = false; + event_fd_.release(); + } + } + EventFd &reader_get_event_fd() { + return event_fd_; + } + void writer_flush() { + //nop + } + + void init() { + event_fd_.init(); + } + void destroy() { + if (!event_fd_.empty()) { + event_fd_.close(); + wait_event_fd_ = false; + writer_vector_.clear(); + reader_vector_.clear(); + reader_pos_ = 0; + } + } + +// Just example of usage +#if !TD_WINDOWS + int reader_wait() { + int res; + + while ((res = reader_wait_nonblock()) == 0) { + // TODO: reader_flush? + pollfd fd; + fd.fd = reader_get_event_fd().get_fd().get_native_fd(); + fd.events = POLLIN; + poll(&fd, 1, -1); + } + return res; + } +#endif + + private: + SpinLock lock_; + bool wait_event_fd_{false}; + EventFd event_fd_; + std::vector<ValueT> writer_vector_; + std::vector<ValueT> reader_vector_; + size_t reader_pos_{0}; +}; + +} // namespace td + +#else +#include "td/utils/logging.h" + +namespace td { + +// dummy implementation which shouldn't be used + +template <class T> +class MpscPollableQueue { + public: + using ValueType = T; + + void init() { + UNREACHABLE(); + } + + template <class PutValueType> + void writer_put(PutValueType &&value) { + UNREACHABLE(); + } + + void writer_flush() { + UNREACHABLE(); + } + + int reader_wait_nonblock() { + UNREACHABLE(); + return 0; + } + + ValueType reader_get_unsafe() { + UNREACHABLE(); + return ValueType(); + } + + void reader_flush() { + UNREACHABLE(); + } + + MpscPollableQueue() = default; + MpscPollableQueue(const MpscPollableQueue &) = delete; + MpscPollableQueue &operator=(const MpscPollableQueue &) = delete; + MpscPollableQueue(MpscPollableQueue &&) = delete; + MpscPollableQueue &operator=(MpscPollableQueue &&) = delete; + ~MpscPollableQueue() = default; +}; + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/Named.h b/libs/tdlib/td/tdutils/td/utils/Named.h new file mode 100644 index 0000000000..202de5f7d4 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Named.h @@ -0,0 +1,27 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +class Named { + public: + Slice get_name() const { + return name_; + } + void set_name(Slice name) { + name_ = name.str(); + } + + private: + string name_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/ObjectPool.h b/libs/tdlib/td/tdutils/td/utils/ObjectPool.h new file mode 100644 index 0000000000..e6e4549dbb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/ObjectPool.h @@ -0,0 +1,249 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <atomic> +#include <memory> +#include <utility> + +namespace td { +// It is draft object pool implementaion +// +// Compared with std::shared_ptr: +// + WeakPtr are much faster. Just pointer copy. No barriers, no atomics. +// - We can't destroy object, because we don't know if it is pointed to by some weak pointer +// +template <class DataT> +class ObjectPool { + struct Storage; + + public: + class WeakPtr { + public: + WeakPtr() : generation_(-1), storage_(nullptr) { + } + WeakPtr(int32 generation, Storage *storage) : generation_(generation), storage_(storage) { + } + + DataT &operator*() const { + return storage_->data; + } + + DataT *operator->() const { + return &**this; + } + + // Pattern of usage: 1. Read an object 2. Check if read was valid via is_alive + // + // It is not very usual case of acquire/release use. + // Instead of publishing an object via some flag we do the opposite. + // We publish new generation via destruction of the data. + // In usual case if we see a flag then we are able to use an object. + // In our case if we have used an object and it is already invalid, then generation will mismatch + bool is_alive() const { + if (!storage_) { + return false; + } + std::atomic_thread_fence(std::memory_order_acquire); + return generation_ == storage_->generation.load(std::memory_order_relaxed); + } + + // Used for ActorId + bool is_alive_unsafe() const { + if (!storage_) { + return false; + } + return generation_ == storage_->generation.load(std::memory_order_relaxed); + } + + bool empty() const { + return storage_ == nullptr; + } + void clear() { + generation_ = -1; + storage_ = nullptr; + } + int32 generation() { + return generation_; + } + + private: + int32 generation_; + Storage *storage_; + }; + + class OwnerPtr { + public: + OwnerPtr() = default; + OwnerPtr(const OwnerPtr &) = delete; + OwnerPtr &operator=(const OwnerPtr &) = delete; + OwnerPtr(OwnerPtr &&other) : storage_(other.storage_), parent_(other.parent_) { + other.storage_ = nullptr; + other.parent_ = nullptr; + } + OwnerPtr &operator=(OwnerPtr &&other) { + if (this != &other) { + storage_ = other.storage_; + parent_ = other.parent_; + other.storage_ = nullptr; + other.parent_ = nullptr; + } + return *this; + } + ~OwnerPtr() { + reset(); + } + + DataT *get() { + return &storage_->data; + } + DataT &operator*() { + return *get(); + } + DataT *operator->() { + return get(); + } + + const DataT *get() const { + return &storage_->data; + } + const DataT &operator*() const { + return *get(); + } + const DataT *operator->() const { + return get(); + } + + WeakPtr get_weak() { + return WeakPtr(storage_->generation.load(std::memory_order_relaxed), storage_); + } + int32 generation() { + return storage_->generation.load(std::memory_order_relaxed); + } + + Storage *release() { + auto result = storage_; + storage_ = nullptr; + return result; + } + + bool empty() const { + return storage_ == nullptr; + } + + void reset() { + if (storage_ != nullptr) { + // for crazy cases when data owns owner pointer to itself. + auto tmp = storage_; + storage_ = nullptr; + parent_->release(OwnerPtr(tmp, parent_)); + } + } + + private: + friend class ObjectPool; + OwnerPtr(Storage *storage, ObjectPool<DataT> *parent) : storage_(storage), parent_(parent) { + } + Storage *storage_ = nullptr; + ObjectPool<DataT> *parent_ = nullptr; + }; + + template <class... ArgsT> + OwnerPtr create(ArgsT &&... args) { + Storage *storage = get_storage(); + storage->init_data(std::forward<ArgsT>(args)...); + return OwnerPtr(storage, this); + } + + OwnerPtr create_empty() { + Storage *storage = get_storage(); + return OwnerPtr(storage, this); + } + + void set_check_empty(bool flag) { + check_empty_flag_ = flag; + } + + void release(OwnerPtr &&owner_ptr) { + Storage *storage = owner_ptr.release(); + storage->destroy_data(); + release_storage(storage); + } + + ObjectPool() = default; + ObjectPool(const ObjectPool &) = delete; + ObjectPool &operator=(const ObjectPool &) = delete; + ObjectPool(ObjectPool &&other) = delete; + ObjectPool &operator=(ObjectPool &&other) = delete; + ~ObjectPool() { + while (head_.load()) { + auto to_delete = head_.load(); + head_ = to_delete->next; + delete to_delete; + storage_count_--; + } + CHECK(storage_count_.load() == 0) << storage_count_.load(); + } + + private: + struct Storage { + // union { + DataT data; + //}; + Storage *next = nullptr; + std::atomic<int32> generation{1}; + + template <class... ArgsT> + void init_data(ArgsT &&... args) { + // new (&data) DataT(std::forward<ArgsT>(args)...); + data = DataT(std::forward<ArgsT>(args)...); + } + void destroy_data() { + generation.fetch_add(1, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_release); + data.clear(); + } + }; + + std::atomic<int32> storage_count_{0}; + std::atomic<Storage *> head_{static_cast<Storage *>(nullptr)}; + bool check_empty_flag_ = false; + + // TODO(perf): allocation Storages in chunks? Anyway we won't be able to release them. + // TODO(perf): memory order + // TODO(perf): use another non lockfree list for release on the same thread + // only one thread, so no aba problem + Storage *get_storage() { + if (head_.load() == nullptr) { + storage_count_++; + return new Storage(); + } + Storage *res; + while (true) { + res = head_.load(); + auto *next = res->next; + if (head_.compare_exchange_weak(res, next)) { + break; + } + } + return res; + } + // release can be called from other thread + void release_storage(Storage *storage) { + while (true) { + auto *save_head = head_.load(); + storage->next = save_head; + if (head_.compare_exchange_weak(save_head, storage)) { + break; + } + } + } +}; +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Observer.h b/libs/tdlib/td/tdutils/td/utils/Observer.h new file mode 100644 index 0000000000..8511e0ce8b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Observer.h @@ -0,0 +1,41 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +namespace td { + +class ObserverBase { + public: + ObserverBase() = default; + ObserverBase(const ObserverBase &) = delete; + ObserverBase &operator=(const ObserverBase &) = delete; + ObserverBase(ObserverBase &&) = delete; + ObserverBase &operator=(ObserverBase &&) = delete; + virtual ~ObserverBase() = default; + + virtual void notify() = 0; +}; + +class Observer : ObserverBase { + public: + Observer() = default; + explicit Observer(unique_ptr<ObserverBase> &&ptr) : observer_ptr_(std::move(ptr)) { + } + + void notify() override { + if (observer_ptr_) { + observer_ptr_->notify(); + } + } + + private: + unique_ptr<ObserverBase> observer_ptr_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/OptionsParser.h b/libs/tdlib/td/tdutils/td/utils/OptionsParser.h new file mode 100644 index 0000000000..6ac0385575 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/OptionsParser.h @@ -0,0 +1,150 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include <functional> +#include <string> + +#if !TD_WINDOWS +#include <getopt.h> +#endif + +namespace td { + +class OptionsParser { + public: + class Option { + public: + enum Type { NoArg, Arg, OptionalArg }; + Type type; + char short_key; + std::string long_key; + std::string description; + std::function<Status(Slice)> arg_callback; + }; + + void set_description(std::string description) { + description_ = std::move(description); + } + + void add_option(Option::Type type, char short_key, Slice long_key, Slice description, + std::function<Status(Slice)> callback) { + options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)}); + } + + void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback) { + add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback)); + } + + void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback) { + // Ouch. There must be some better way + add_option(Option::Type::NoArg, short_key, long_key, description, + std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback), + std::placeholders::_1)); + } + + Result<int> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT { +#if TD_WINDOWS + return -1; +#else + // use getopt. long keys are not supported for now + char buff[1024]; + StringBuilder sb({buff, sizeof(buff)}); + for (auto &opt : options_) { + CHECK(opt.type != Option::OptionalArg); + sb << opt.short_key; + if (opt.type == Option::Arg) { + sb << ":"; + } + } + if (sb.is_error()) { + return Status::Error("Can't parse options"); + } + CSlice short_options = sb.as_cslice(); + + vector<option> long_options; + for (auto &opt : options_) { + if (opt.long_key.empty()) { + continue; + } + option o; + o.flag = nullptr; + o.val = opt.short_key; + o.has_arg = opt.type == Option::Arg ? required_argument : no_argument; + o.name = opt.long_key.c_str(); + long_options.push_back(o); + } + long_options.push_back({nullptr, 0, nullptr, 0}); + + while (true) { + int opt_i = getopt_long(argc, argv, short_options.c_str(), &long_options[0], nullptr); + if (opt_i == ':') { + return Status::Error("Missing argument"); + } + if (opt_i == '?') { + return Status::Error("Unrecognized option"); + } + if (opt_i == -1) { + break; + } + bool found = false; + for (auto &opt : options_) { + if (opt.short_key == opt_i) { + Slice arg; + if (opt.type == Option::Arg) { + arg = Slice(optarg); + } + auto status = opt.arg_callback(arg); + if (status.is_error()) { + return std::move(status); + } + found = true; + break; + } + } + if (!found) { + return Status::Error("Unknown argument"); + } + } + return optind; +#endif + } + + friend StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o) { + sb << o.description_ << "\n"; + for (auto &opt : o.options_) { + sb << "-" << opt.short_key; + if (!opt.long_key.empty()) { + sb << "|--" << opt.long_key; + } + if (opt.type == Option::OptionalArg) { + sb << "["; + } + if (opt.type != Option::NoArg) { + sb << "<arg>"; + } + if (opt.type == Option::OptionalArg) { + sb << "]"; + } + sb << "\t" << opt.description; + sb << "\n"; + } + return sb; + } + + private: + std::vector<Option> options_; + std::string description_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/OrderedEventsProcessor.h b/libs/tdlib/td/tdutils/td/utils/OrderedEventsProcessor.h new file mode 100644 index 0000000000..4515b74684 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/OrderedEventsProcessor.h @@ -0,0 +1,87 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <utility> + +namespace td { + +// Process states in order defined by their Id +template <class DataT> +class OrderedEventsProcessor { + public: + using SeqNo = uint64; + + OrderedEventsProcessor() = default; + explicit OrderedEventsProcessor(SeqNo offset) : offset_(offset), begin_(offset_), end_(offset_) { + } + + template <class FromDataT, class FunctionT> + void add(SeqNo seq_no, FromDataT &&data, FunctionT &&function) { + CHECK(seq_no >= begin_) << seq_no << ">=" << begin_; // or ignore? + + if (seq_no == begin_) { // run now + begin_++; + function(seq_no, std::forward<FromDataT>(data)); + + while (begin_ < end_) { + auto &data_flag = data_array_[static_cast<size_t>(begin_ - offset_)]; + if (!data_flag.second) { + break; + } + function(begin_, std::move(data_flag.first)); + data_flag.second = false; + begin_++; + } + if (begin_ > end_) { + end_ = begin_; + } + if (begin_ == end_) { + offset_ = begin_; + } + + // try_compactify + auto begin_pos = static_cast<size_t>(begin_ - offset_); + if (begin_pos > 5 && begin_pos * 2 > data_array_.size()) { + data_array_.erase(data_array_.begin(), data_array_.begin() + begin_pos); + offset_ = begin_; + } + } else { + auto pos = static_cast<size_t>(seq_no - offset_); + auto need_size = pos + 1; + if (data_array_.size() < need_size) { + data_array_.resize(need_size); + } + data_array_[pos].first = std::forward<FromDataT>(data); + data_array_[pos].second = true; + if (end_ < seq_no + 1) { + end_ = seq_no + 1; + } + } + } + + bool has_events() const { + return begin_ != end_; + } + SeqNo max_unfinished_seq_no() { + return end_ - 1; + } + SeqNo max_finished_seq_no() { + return begin_ - 1; + } + + private: + SeqNo offset_ = 1; + SeqNo begin_ = 1; + SeqNo end_ = 1; + std::vector<std::pair<DataT, bool>> data_array_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Parser.h b/libs/tdlib/td/tdutils/td/utils/Parser.h new file mode 100644 index 0000000000..06e95bf807 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Parser.h @@ -0,0 +1,183 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <cstring> +#include <utility> + +namespace td { + +class Parser { + public: + explicit Parser(MutableSlice data) : ptr_(data.begin()), end_(data.end()), status_() { + } + Parser(Parser &&other) : ptr_(other.ptr_), end_(other.end_), status_(std::move(other.status_)) { + other.clear(); + } + Parser &operator=(Parser &&other) { + if (&other == this) { + return *this; + } + ptr_ = other.ptr_; + end_ = other.end_; + status_ = std::move(other.status_); + other.clear(); + return *this; + } + Parser(const Parser &) = delete; + Parser &operator=(const Parser &) = delete; + ~Parser() = default; + + bool empty() const { + return ptr_ == end_; + } + void clear() { + ptr_ = nullptr; + end_ = ptr_; + status_ = Status::OK(); + } + + MutableSlice read_till_nofail(char c) { + if (status_.is_error()) { + return MutableSlice(); + } + char *till = reinterpret_cast<char *>(std::memchr(ptr_, c, end_ - ptr_)); + if (till == nullptr) { + till = end_; + } + MutableSlice result(ptr_, till); + ptr_ = till; + return result; + } + + MutableSlice read_till_nofail(Slice str) { + if (status_.is_error()) { + return MutableSlice(); + } + char *best_till = end_; + for (auto c : str) { + char *till = reinterpret_cast<char *>(std::memchr(ptr_, c, end_ - ptr_)); + if (till != nullptr && till < best_till) { + best_till = till; + } + } + MutableSlice result(ptr_, best_till); + ptr_ = best_till; + return result; + } + + template <class F> + MutableSlice read_while(const F &f) { + auto save_ptr = ptr_; + while (ptr_ != end_ && f(*ptr_)) { + ptr_++; + } + return MutableSlice(save_ptr, ptr_); + } + MutableSlice read_all() { + auto save_ptr = ptr_; + ptr_ = end_; + return MutableSlice(save_ptr, ptr_); + } + + MutableSlice read_till(char c) { + if (status_.is_error()) { + return MutableSlice(); + } + MutableSlice res = read_till_nofail(c); + if (ptr_ == end_ || ptr_[0] != c) { + status_ = Status::Error(PSLICE() << "Read till " << tag("char", c) << " failed"); + return MutableSlice(); + } + return res; + } + + char peek_char() { + if (ptr_ == end_) { + return 0; + } + return *ptr_; + } + + char *ptr() { + return ptr_; + } + + void skip_nofail(char c) { + if (ptr_ != end_ && ptr_[0] == c) { + ptr_++; + } + } + void skip(char c) { + if (status_.is_error()) { + return; + } + if (ptr_ == end_ || ptr_[0] != c) { + status_ = Status::Error(PSLICE() << "Skip " << tag("char", c) << " failed"); + return; + } + ptr_++; + } + bool try_skip(char c) { + if (ptr_ != end_ && ptr_[0] == c) { + ptr_++; + return true; + } + return false; + } + + void skip_till_not(Slice str) { + while (ptr_ != end_) { + if (std::memchr(str.data(), *ptr_, str.size()) == nullptr) { + break; + } + ptr_++; + } + } + void skip_whitespaces() { + skip_till_not(" \t\r\n"); + } + + MutableSlice data() const { + return MutableSlice(ptr_, end_); + } + + Status &status() { + return status_; + } + + bool start_with(Slice prefix) { + if (prefix.size() + ptr_ > end_) { + return false; + } + return std::memcmp(prefix.begin(), ptr_, prefix.size()) == 0; + } + + bool skip_start_with(Slice prefix) { + if (start_with(prefix)) { + advance(prefix.size()); + return true; + } + return false; + } + + void advance(size_t diff) { + ptr_ += diff; + CHECK(ptr_ <= end_); + } + + private: + char *ptr_; + char *end_; + Status status_; +}; +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/PathView.h b/libs/tdlib/td/tdutils/td/utils/PathView.h new file mode 100644 index 0000000000..edb5d7c127 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/PathView.h @@ -0,0 +1,116 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/misc.h" +#include "td/utils/Slice.h" + +namespace td { + +class PathView { + public: + explicit PathView(Slice path) : path_(path) { + last_slash_ = narrow_cast<int32>(path_.size()) - 1; + while (last_slash_ >= 0 && !is_slash(path_[last_slash_])) { + last_slash_--; + } + + last_dot_ = static_cast<int32>(path_.size()); + for (auto i = last_dot_ - 1; i > last_slash_ + 1; i--) { + if (path_[i] == '.') { + last_dot_ = i; + break; + } + } + } + + bool empty() const { + return path_.empty(); + } + + bool is_dir() const { + if (empty()) { + return false; + } + return is_slash(path_.back()); + } + + Slice parent_dir() const { + return Slice(path_.begin(), last_slash_ + 1); + } + + Slice extension() const { + if (last_dot_ == static_cast<int32>(path_.size())) { + return Slice(); + } + return path_.substr(last_dot_ + 1); + } + + Slice without_extension() const { + return Slice(path_.begin(), last_dot_); + } + + Slice file_stem() const { + return path_.substr(last_slash_ + 1, last_dot_ - last_slash_ - 1); + } + + Slice file_name() const { + return path_.substr(last_slash_ + 1); + } + + Slice path() const { + return path_; + } + + bool is_absolute() const { + return !empty() && (is_slash(path_[0]) || (path_.size() >= 3 && path_[1] == ':' && is_slash(path_[2]))); + } + + bool is_relative() const { + return !is_absolute(); + } + + static Slice relative(Slice path, Slice dir, bool force = false) { + if (begins_with(path, dir)) { + path.remove_prefix(dir.size()); + return path; + } + if (force) { + return Slice(); + } + return path; + } + + static Slice dir_and_file(Slice path) { + auto last_slash = static_cast<int32>(path.size()) - 1; + while (last_slash >= 0 && !is_slash(path[last_slash])) { + last_slash--; + } + if (last_slash < 0) { + return Slice(); + } + last_slash--; + while (last_slash >= 0 && !is_slash(path[last_slash])) { + last_slash--; + } + if (last_slash < 0) { + return Slice(); + } + return path.substr(last_slash + 1); + } + + private: + static bool is_slash(char c) { + return c == '/' || c == '\\'; + } + + Slice path_; + int32 last_slash_; + int32 last_dot_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Random.cpp b/libs/tdlib/td/tdutils/td/utils/Random.cpp new file mode 100644 index 0000000000..db11df4dfa --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Random.cpp @@ -0,0 +1,108 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Random.h" + +#include "td/utils/logging.h" +#include "td/utils/port/thread_local.h" + +#if TD_HAVE_OPENSSL +#include <openssl/rand.h> +#endif + +#include <cstring> +#include <limits> +#include <random> + +namespace td { + +#if TD_HAVE_OPENSSL +void Random::secure_bytes(MutableSlice dest) { + Random::secure_bytes(dest.ubegin(), dest.size()); +} + +void Random::secure_bytes(unsigned char *ptr, size_t size) { + constexpr size_t buf_size = 512; + static TD_THREAD_LOCAL unsigned char *buf; // static zero-initialized + static TD_THREAD_LOCAL size_t buf_pos; + if (init_thread_local<unsigned char[]>(buf, buf_size)) { + buf_pos = buf_size; + } + + auto ready = min(size, buf_size - buf_pos); + if (ready != 0) { + std::memcpy(ptr, buf + buf_pos, ready); + buf_pos += ready; + ptr += ready; + size -= ready; + if (size == 0) { + return; + } + } + if (size < buf_size) { + int err = RAND_bytes(buf, static_cast<int>(buf_size)); + // TODO: it CAN fail + LOG_IF(FATAL, err != 1); + buf_pos = size; + std::memcpy(ptr, buf, size); + return; + } + + CHECK(size <= static_cast<size_t>(std::numeric_limits<int>::max())); + int err = RAND_bytes(ptr, static_cast<int>(size)); + // TODO: it CAN fail + LOG_IF(FATAL, err != 1); +} + +int32 Random::secure_int32() { + int32 res = 0; + secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(int32)); + return res; +} + +int64 Random::secure_int64() { + int64 res = 0; + secure_bytes(reinterpret_cast<unsigned char *>(&res), sizeof(int64)); + return res; +} +#endif + +static unsigned int rand_device_helper() { + static TD_THREAD_LOCAL std::random_device *rd; + init_thread_local<std::random_device>(rd); + return (*rd)(); +} + +uint32 Random::fast_uint32() { + static TD_THREAD_LOCAL std::mt19937 *gen; + if (!gen) { + auto &rg = rand_device_helper; + std::seed_seq seq{rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg()}; + init_thread_local<std::mt19937>(gen, seq); + } + return static_cast<uint32>((*gen)()); +} + +uint64 Random::fast_uint64() { + static TD_THREAD_LOCAL std::mt19937_64 *gen; + if (!gen) { + auto &rg = rand_device_helper; + std::seed_seq seq{rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg(), rg()}; + init_thread_local<std::mt19937_64>(gen, seq); + } + return static_cast<uint64>((*gen)()); +} + +int Random::fast(int min, int max) { + if (min == std::numeric_limits<int>::min() && max == std::numeric_limits<int>::max()) { + // to prevent integer overflow and division by zero + min++; + } + CHECK(min <= max); + return static_cast<int>(min + fast_uint32() % (max - min + 1)); // TODO signed_cast +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Random.h b/libs/tdlib/td/tdutils/td/utils/Random.h new file mode 100644 index 0000000000..efe5d64618 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Random.h @@ -0,0 +1,30 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +class Random { + public: +#if TD_HAVE_OPENSSL + static void secure_bytes(MutableSlice dest); + static void secure_bytes(unsigned char *ptr, size_t size); + static int32 secure_int32(); + static int64 secure_int64(); +#endif + + static uint32 fast_uint32(); + static uint64 fast_uint64(); + + // distribution is not uniform, min and max are included + static int fast(int min, int max); +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/ScopeGuard.h b/libs/tdlib/td/tdutils/td/utils/ScopeGuard.h new file mode 100644 index 0000000000..a914ce357c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/ScopeGuard.h @@ -0,0 +1,76 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include <cstdlib> +#include <memory> +#include <type_traits> +#include <utility> + +namespace td { +class Guard { + public: + Guard() = default; + Guard(const Guard &other) = delete; + Guard &operator=(const Guard &other) = delete; + Guard(Guard &&other) = default; + Guard &operator=(Guard &&other) = default; + virtual ~Guard() = default; + virtual void dismiss() { + std::abort(); + } +}; + +template <class FunctionT> +class LambdaGuard : public Guard { + public: + explicit LambdaGuard(const FunctionT &func) : func_(func) { + } + explicit LambdaGuard(FunctionT &&func) : func_(std::move(func)) { + } + LambdaGuard(const LambdaGuard &other) = delete; + LambdaGuard &operator=(const LambdaGuard &other) = delete; + LambdaGuard(LambdaGuard &&other) : func_(std::move(other.func_)), dismissed_(other.dismissed_) { + other.dismissed_ = true; + } + LambdaGuard &operator=(LambdaGuard &&other) = delete; + + void dismiss() { + dismissed_ = true; + } + + ~LambdaGuard() { + if (!dismissed_) { + func_(); + } + } + + private: + FunctionT func_; + bool dismissed_ = false; +}; + +template <class F> +std::unique_ptr<Guard> create_lambda_guard(F &&f) { + return std::make_unique<LambdaGuard<F>>(std::forward<F>(f)); +} +template <class F> +std::shared_ptr<Guard> create_shared_lambda_guard(F &&f) { + return std::make_shared<LambdaGuard<F>>(std::forward<F>(f)); +} + +enum class ScopeExit {}; +template <class FunctionT> +auto operator+(ScopeExit, FunctionT &&func) { + return LambdaGuard<std::decay_t<FunctionT>>(std::forward<FunctionT>(func)); +} + +} // namespace td + +#define SCOPE_EXIT auto TD_CONCAT(SCOPE_EXIT_VAR_, __LINE__) = ::td::ScopeExit() + [&]() diff --git a/libs/tdlib/td/tdutils/td/utils/SharedObjectPool.h b/libs/tdlib/td/tdutils/td/utils/SharedObjectPool.h new file mode 100644 index 0000000000..dc8512b268 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/SharedObjectPool.h @@ -0,0 +1,276 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include "td/utils/logging.h" +#include "td/utils/MpscLinkQueue.h" + +#include <atomic> +#include <memory> +#include <new> +#include <utility> + +namespace td { + +namespace detail { +class AtomicRefCnt { + public: + explicit AtomicRefCnt(uint64 cnt) : cnt_(cnt) { + } + void inc() { + cnt_.fetch_add(1, std::memory_order_relaxed); + } + bool dec() { + return cnt_.fetch_sub(1, std::memory_order_acq_rel) == 1; + } + uint64 value() const { + return cnt_.load(std::memory_order_relaxed); + } + + private: + std::atomic<uint64> cnt_; +}; + +template <class DataT, class DeleterT> +class SharedPtrRaw + : public DeleterT + , private MpscLinkQueueImpl::Node { + public: + explicit SharedPtrRaw(DeleterT deleter) : DeleterT(std::move(deleter)), ref_cnt_{0}, option_magic_(Magic) { + } + + ~SharedPtrRaw() { + CHECK(use_cnt() == 0); + CHECK(option_magic_ == Magic); + } + template <class... ArgsT> + void init_data(ArgsT &&... args) { + new (&option_data_) DataT(std::forward<ArgsT>(args)...); + } + void destroy_data() { + option_data_.~DataT(); + option_magic_ = Magic; + } + uint64 use_cnt() const { + return ref_cnt_.value(); + } + void inc() { + ref_cnt_.inc(); + } + bool dec() { + return ref_cnt_.dec(); + } + DataT &data() { + return option_data_; + } + static SharedPtrRaw *from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) { + return static_cast<SharedPtrRaw<DataT, DeleterT> *>(node); + } + MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() { + return static_cast<MpscLinkQueueImpl::Node *>(this); + } + + private: + AtomicRefCnt ref_cnt_; + enum { Magic = 0x732817a2 }; + union { + DataT option_data_; + uint32 option_magic_; + }; +}; + +template <class T, class DeleterT = std::default_delete<T>> +class SharedPtr { + public: + using Raw = detail::SharedPtrRaw<T, DeleterT>; + SharedPtr() = default; + ~SharedPtr() { + if (!raw_) { + return; + } + reset(); + } + explicit SharedPtr(Raw *raw) : raw_(raw) { + raw_->inc(); + } + SharedPtr(const SharedPtr &other) : SharedPtr(other.raw_) { + } + SharedPtr &operator=(const SharedPtr &other) { + other.raw_->inc(); + reset(other.raw_); + return *this; + } + SharedPtr(SharedPtr &&other) : raw_(other.raw_) { + other.raw_ = nullptr; + } + SharedPtr &operator=(SharedPtr &&other) { + reset(other.raw_); + other.raw_ = nullptr; + return *this; + } + bool empty() const { + return raw_ == nullptr; + } + explicit operator bool() const { + return !empty(); + } + uint64 use_cnt() const { + if (!raw_) { + return 0; + } + return raw_->use_cnt(); + } + T &operator*() const { + return raw_->data(); + } + T *operator->() const { + return &raw_->data(); + } + + Raw *release() { + auto res = raw_; + raw_ = nullptr; + return res; + } + + void reset(Raw *new_raw = nullptr) { + if (raw_ && raw_->dec()) { + raw_->destroy_data(); + auto deleter = std::move(static_cast<DeleterT &>(*raw_)); + deleter(raw_); + } + raw_ = new_raw; + } + + template <class... ArgsT> + static SharedPtr<T, DeleterT> create(ArgsT &&... args) { + auto raw = std::make_unique<Raw>(DeleterT()); + raw->init_data(std::forward<ArgsT>(args)...); + return SharedPtr<T, DeleterT>(raw.release()); + } + template <class D, class... ArgsT> + static SharedPtr<T, DeleterT> create_with_deleter(D &&d, ArgsT &&... args) { + auto raw = std::make_unique<Raw>(std::forward<D>(d)); + raw->init_data(std::forward<ArgsT>(args)...); + return SharedPtr<T, DeleterT>(raw.release()); + } + + private: + Raw *raw_{nullptr}; +}; + +} // namespace detail + +template <class DataT> +class SharedObjectPool { + class Deleter; + + public: + using Ptr = detail::SharedPtr<DataT, Deleter>; + + SharedObjectPool() = default; + SharedObjectPool(const SharedObjectPool &other) = delete; + SharedObjectPool &operator=(const SharedObjectPool &other) = delete; + SharedObjectPool(SharedObjectPool &&other) = delete; + SharedObjectPool &operator=(SharedObjectPool &&other) = delete; + ~SharedObjectPool() { + free_queue_.pop_all(free_queue_reader_); + size_t free_cnt = 0; + while (free_queue_reader_.read()) { + free_cnt++; + } + CHECK(free_cnt == allocated_.size()) << free_cnt << " " << allocated_.size(); + } + + template <class... ArgsT> + Ptr alloc(ArgsT &&... args) { + auto *raw = alloc_raw(); + raw->init_data(std::forward<ArgsT>(args)...); + return Ptr(raw); + } + size_t total_size() const { + return allocated_.size(); + } + uint64 calc_free_size() { + free_queue_.pop_all(free_queue_reader_); + return free_queue_reader_.calc_size(); + } + + //non thread safe + template <class F> + void for_each(F &&f) { + for (auto &raw : allocated_) { + if (raw->use_cnt() > 0) { + f(raw->data()); + } + } + } + + private: + using Raw = typename Ptr::Raw; + Raw *alloc_raw() { + free_queue_.pop_all(free_queue_reader_); + auto *raw = free_queue_reader_.read().get(); + if (raw) { + return raw; + } + allocated_.push_back(std::make_unique<Raw>(deleter())); + return allocated_.back().get(); + } + + void free_raw(Raw *raw) { + free_queue_.push(Node{raw}); + } + + class Node { + public: + Node() = default; + explicit Node(Raw *raw) : raw_(raw) { + } + + MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() { + return raw_->to_mpsc_link_queue_node(); + } + static Node from_mpsc_link_queue_node(MpscLinkQueueImpl::Node *node) { + return Node{Raw::from_mpsc_link_queue_node(node)}; + } + Raw *get() const { + return raw_; + } + explicit operator bool() const { + return raw_ != nullptr; + } + + private: + Raw *raw_{nullptr}; + }; + + class Deleter { + public: + explicit Deleter(SharedObjectPool<DataT> *pool) : pool_(pool) { + } + void operator()(Raw *raw) { + pool_->free_raw(raw); + }; + + private: + SharedObjectPool<DataT> *pool_; + }; + friend class Deleter; + + Deleter deleter() { + return Deleter(this); + } + + std::vector<std::unique_ptr<Raw>> allocated_; + MpscLinkQueue<Node> free_queue_; + typename MpscLinkQueue<Node>::Reader free_queue_reader_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Slice-decl.h b/libs/tdlib/td/tdutils/td/utils/Slice-decl.h new file mode 100644 index 0000000000..69b4a4ad21 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Slice-decl.h @@ -0,0 +1,187 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include <cstring> +#include <type_traits> + +namespace td { +class Slice; + +class MutableSlice { + char *s_; + size_t len_; + + struct private_tag {}; + + public: + MutableSlice(); + MutableSlice(char *s, size_t len); + MutableSlice(unsigned char *s, size_t len); + MutableSlice(string &s); + template <class T> + explicit MutableSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag> = {}) + : MutableSlice(s, std::strlen(s)) { + } + MutableSlice(char *s, char *t); + MutableSlice(unsigned char *s, unsigned char *t); + template <size_t N> + constexpr MutableSlice(char (&a)[N]) = delete; + + bool empty() const; + size_t size() const; + + MutableSlice &remove_prefix(size_t prefix_len); + MutableSlice &remove_suffix(size_t suffix_len); + MutableSlice &truncate(size_t size); + + MutableSlice copy() const; + + char *data() const; + char *begin() const; + unsigned char *ubegin() const; + char *end() const; + unsigned char *uend() const; + + string str() const; + MutableSlice substr(size_t from) const; + MutableSlice substr(size_t from, size_t size) const; + size_t find(char c) const; + size_t rfind(char c) const; + + void copy_from(Slice from); + + char &back(); + char &operator[](size_t i); +}; + +class Slice { + const char *s_; + size_t len_; + + struct private_tag {}; + + public: + Slice(); + Slice(const MutableSlice &other); + Slice(const char *s, size_t len); + Slice(const unsigned char *s, size_t len); + Slice(const string &s); + template <class T> + explicit Slice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag> = {}) + : Slice(s, std::strlen(s)) { + } + template <class T> + explicit Slice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag> = {}) + : Slice(s, std::strlen(s)) { + } + Slice(const char *s, const char *t); + Slice(const unsigned char *s, const unsigned char *t); + + template <size_t N> + constexpr Slice(char (&a)[N]) = delete; + + template <size_t N> + constexpr Slice(const char (&a)[N]) : s_(a), len_(N - 1) { + } + + bool empty() const; + size_t size() const; + + Slice &remove_prefix(size_t prefix_len); + Slice &remove_suffix(size_t suffix_len); + Slice &truncate(size_t size); + + Slice copy() const; + + const char *data() const; + const char *begin() const; + const unsigned char *ubegin() const; + const char *end() const; + const unsigned char *uend() const; + + string str() const; + Slice substr(size_t from) const; + Slice substr(size_t from, size_t size) const; + size_t find(char c) const; + size_t rfind(char c) const; + + char back() const; + char operator[](size_t i) const; +}; + +bool operator==(const Slice &a, const Slice &b); +bool operator!=(const Slice &a, const Slice &b); + +class MutableCSlice : public MutableSlice { + struct private_tag {}; + + MutableSlice &remove_suffix(size_t suffix_len) = delete; + MutableSlice &truncate(size_t size) = delete; + + public: + MutableCSlice() = delete; + MutableCSlice(string &s) : MutableSlice(s) { + } + template <class T> + explicit MutableCSlice(T s, std::enable_if_t<std::is_same<char *, T>::value, private_tag> = {}) : MutableSlice(s) { + } + MutableCSlice(char *s, char *t); + + template <size_t N> + constexpr MutableCSlice(char (&a)[N]) = delete; + + const char *c_str() const { + return begin(); + } +}; + +class CSlice : public Slice { + struct private_tag {}; + + Slice &remove_suffix(size_t suffix_len) = delete; + Slice &truncate(size_t size) = delete; + + public: + explicit CSlice(const MutableSlice &other) : Slice(other) { + } + CSlice(const MutableCSlice &other) : Slice(other.begin(), other.size()) { + } + CSlice(const string &s) : Slice(s) { + } + template <class T> + explicit CSlice(T s, std::enable_if_t<std::is_same<char *, std::remove_const_t<T>>::value, private_tag> = {}) + : Slice(s) { + } + template <class T> + explicit CSlice(T s, std::enable_if_t<std::is_same<const char *, std::remove_const_t<T>>::value, private_tag> = {}) + : Slice(s) { + } + CSlice(const char *s, const char *t); + + template <size_t N> + constexpr CSlice(char (&a)[N]) = delete; + + template <size_t N> + constexpr CSlice(const char (&a)[N]) : Slice(a) { + } + + CSlice() : CSlice("") { + } + + const char *c_str() const { + return begin(); + } +}; + +struct SliceHash { + std::size_t operator()(Slice slice) const; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Slice.h b/libs/tdlib/td/tdutils/td/utils/Slice.h new file mode 100644 index 0000000000..a9bc6a7551 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Slice.h @@ -0,0 +1,275 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Slice-decl.h" + +#include "td/utils/logging.h" + +#include <cstring> + +namespace td { +/*** MutableSlice ***/ +inline MutableSlice::MutableSlice() : MutableSlice(const_cast<char *>(""), static_cast<size_t>(0)) { +} + +inline MutableSlice::MutableSlice(char *s, size_t len) : s_(s), len_(len) { + CHECK(s_ != nullptr); +} + +inline MutableSlice::MutableSlice(unsigned char *s, size_t len) : s_(reinterpret_cast<char *>(s)), len_(len) { + CHECK(s_ != nullptr); +} + +inline MutableSlice::MutableSlice(string &s) : MutableSlice(&s[0], s.size()) { +} + +inline MutableSlice::MutableSlice(char *s, char *t) : MutableSlice(s, t - s) { +} + +inline MutableSlice::MutableSlice(unsigned char *s, unsigned char *t) : MutableSlice(s, t - s) { +} + +inline size_t MutableSlice::size() const { + return len_; +} + +inline MutableSlice &MutableSlice::remove_prefix(size_t prefix_len) { + CHECK(prefix_len <= len_); + s_ += prefix_len; + len_ -= prefix_len; + return *this; +} +inline MutableSlice &MutableSlice::remove_suffix(size_t suffix_len) { + CHECK(suffix_len <= len_); + len_ -= suffix_len; + return *this; +} + +inline MutableSlice &MutableSlice::truncate(size_t size) { + if (len_ > size) { + len_ = size; + } + return *this; +} + +inline MutableSlice MutableSlice::copy() const { + return *this; +} + +inline bool MutableSlice::empty() const { + return len_ == 0; +} + +inline char *MutableSlice::data() const { + return s_; +} + +inline char *MutableSlice::begin() const { + return s_; +} + +inline unsigned char *MutableSlice::ubegin() const { + return reinterpret_cast<unsigned char *>(s_); +} + +inline char *MutableSlice::end() const { + return s_ + len_; +} + +inline unsigned char *MutableSlice::uend() const { + return reinterpret_cast<unsigned char *>(s_) + len_; +} + +inline string MutableSlice::str() const { + return string(begin(), size()); +} + +inline MutableSlice MutableSlice::substr(size_t from) const { + CHECK(from <= len_); + return MutableSlice(s_ + from, len_ - from); +} +inline MutableSlice MutableSlice::substr(size_t from, size_t size) const { + CHECK(from <= len_); + return MutableSlice(s_ + from, min(size, len_ - from)); +} + +inline size_t MutableSlice::find(char c) const { + for (size_t pos = 0; pos < len_; pos++) { + if (s_[pos] == c) { + return pos; + } + } + return static_cast<size_t>(-1); +} + +inline size_t MutableSlice::rfind(char c) const { + for (size_t pos = len_; pos-- > 0;) { + if (s_[pos] == c) { + return pos; + } + } + return static_cast<size_t>(-1); +} + +inline void MutableSlice::copy_from(Slice from) { + CHECK(size() >= from.size()); + std::memcpy(ubegin(), from.ubegin(), from.size()); +} + +inline char &MutableSlice::back() { + CHECK(1 <= len_); + return s_[len_ - 1]; +} + +inline char &MutableSlice::operator[](size_t i) { + return s_[i]; +} + +/*** Slice ***/ +inline Slice::Slice() : Slice("", static_cast<size_t>(0)) { +} + +inline Slice::Slice(const MutableSlice &other) : Slice(other.begin(), other.size()) { +} + +inline Slice::Slice(const char *s, size_t len) : s_(s), len_(len) { + CHECK(s_ != nullptr); +} + +inline Slice::Slice(const unsigned char *s, size_t len) : s_(reinterpret_cast<const char *>(s)), len_(len) { + CHECK(s_ != nullptr); +} + +inline Slice::Slice(const string &s) : Slice(s.c_str(), s.size()) { +} + +inline Slice::Slice(const char *s, const char *t) : Slice(s, t - s) { +} + +inline Slice::Slice(const unsigned char *s, const unsigned char *t) : Slice(s, t - s) { +} + +inline size_t Slice::size() const { + return len_; +} + +inline Slice &Slice::remove_prefix(size_t prefix_len) { + CHECK(prefix_len <= len_); + s_ += prefix_len; + len_ -= prefix_len; + return *this; +} + +inline Slice &Slice::remove_suffix(size_t suffix_len) { + CHECK(suffix_len <= len_); + len_ -= suffix_len; + return *this; +} + +inline Slice &Slice::truncate(size_t size) { + if (len_ > size) { + len_ = size; + } + return *this; +} + +inline Slice Slice::copy() const { + return *this; +} + +inline bool Slice::empty() const { + return len_ == 0; +} + +inline const char *Slice::data() const { + return s_; +} + +inline const char *Slice::begin() const { + return s_; +} + +inline const unsigned char *Slice::ubegin() const { + return reinterpret_cast<const unsigned char *>(s_); +} + +inline const char *Slice::end() const { + return s_ + len_; +} + +inline const unsigned char *Slice::uend() const { + return reinterpret_cast<const unsigned char *>(s_) + len_; +} + +inline string Slice::str() const { + return string(begin(), size()); +} + +inline Slice Slice::substr(size_t from) const { + CHECK(from <= len_); + return Slice(s_ + from, len_ - from); +} +inline Slice Slice::substr(size_t from, size_t size) const { + CHECK(from <= len_); + return Slice(s_ + from, min(size, len_ - from)); +} + +inline size_t Slice::find(char c) const { + for (size_t pos = 0; pos < len_; pos++) { + if (s_[pos] == c) { + return pos; + } + } + return static_cast<size_t>(-1); +} + +inline size_t Slice::rfind(char c) const { + for (size_t pos = len_; pos-- > 0;) { + if (s_[pos] == c) { + return pos; + } + } + return static_cast<size_t>(-1); +} + +inline char Slice::back() const { + CHECK(1 <= len_); + return s_[len_ - 1]; +} + +inline char Slice::operator[](size_t i) const { + return s_[i]; +} + +inline bool operator==(const Slice &a, const Slice &b) { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size()) == 0; +} + +inline bool operator!=(const Slice &a, const Slice &b) { + return !(a == b); +} + +inline MutableCSlice::MutableCSlice(char *s, char *t) : MutableSlice(s, t) { + CHECK(*t == '\0'); +} + +inline CSlice::CSlice(const char *s, const char *t) : Slice(s, t) { + CHECK(*t == '\0'); +} + +inline std::size_t SliceHash::operator()(Slice slice) const { + // simple string hash + std::size_t result = 0; + constexpr std::size_t MUL = 123456789; + for (auto c : slice) { + result = result * MUL + c; + } + return result; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/SpinLock.h b/libs/tdlib/td/tdutils/td/utils/SpinLock.h new file mode 100644 index 0000000000..d726b0b2f6 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/SpinLock.h @@ -0,0 +1,58 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/thread.h" + +#include <atomic> + +namespace td { + +class SpinLock { + struct Unlock { + void operator()(SpinLock *ptr) { + ptr->unlock(); + } + }; + + class InfBackoff { + int cnt = 0; + + public: + bool next() { + cnt++; + if (cnt < 50) { + return true; + } else { + td::this_thread::yield(); + return true; + } + } + }; + + public: + using Lock = std::unique_ptr<SpinLock, Unlock>; + + Lock lock() { + InfBackoff backoff; + while (!try_lock()) { + backoff.next(); + } + return Lock(this); + } + bool try_lock() { + return !flag_.test_and_set(std::memory_order_acquire); + } + + private: + std::atomic_flag flag_ = ATOMIC_FLAG_INIT; + void unlock() { + flag_.clear(std::memory_order_release); + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/StackAllocator.cpp b/libs/tdlib/td/tdutils/td/utils/StackAllocator.cpp new file mode 100644 index 0000000000..4db905368b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/StackAllocator.cpp @@ -0,0 +1,18 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/StackAllocator.h" + +#include "td/utils/port/thread_local.h" + +namespace td { + +StackAllocator::Impl &StackAllocator::impl() { + static TD_THREAD_LOCAL StackAllocator::Impl *impl; // static zero-initialized + init_thread_local<Impl>(impl); + return *impl; +} +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/StackAllocator.h b/libs/tdlib/td/tdutils/td/utils/StackAllocator.h new file mode 100644 index 0000000000..d2399b9526 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/StackAllocator.h @@ -0,0 +1,82 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/MovableValue.h" +#include "td/utils/Slice-decl.h" + +#include <array> +#include <cstdlib> + +namespace td { + +class StackAllocator { + class Deleter { + public: + void operator()(char *ptr) { + free_ptr(ptr); + } + }; + + // TODO: alloc memory with mmap and unload unused pages + // memory still can be corrupted, but it is better than explicit free function + // TODO: use pointer that can't be even copied + using PtrImpl = std::unique_ptr<char, Deleter>; + class Ptr { + public: + Ptr(char *ptr, size_t size) : ptr_(ptr), size_(size) { + } + + MutableSlice as_slice() const { + return MutableSlice(ptr_.get(), size_.get()); + } + + private: + PtrImpl ptr_; + MovableValue<size_t> size_; + }; + + static void free_ptr(char *ptr) { + impl().free_ptr(ptr); + } + + struct Impl { + static const size_t MEM_SIZE = 1024 * 1024; + std::array<char, MEM_SIZE> mem; + + size_t pos{0}; + char *alloc(size_t size) { + if (size == 0) { + size = 1; + } + char *res = mem.data() + pos; + size = (size + 7) & -8; + pos += size; + if (pos > MEM_SIZE) { + std::abort(); // memory is over + } + return res; + } + void free_ptr(char *ptr) { + size_t new_pos = ptr - mem.data(); + if (new_pos >= pos) { + std::abort(); // shouldn't happen + } + pos = new_pos; + } + }; + + static Impl &impl(); + + public: + static Ptr alloc(size_t size) { + return Ptr(impl().alloc(size), size); + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Status.cpp b/libs/tdlib/td/tdutils/td/utils/Status.cpp new file mode 100644 index 0000000000..b8bb169e60 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Status.cpp @@ -0,0 +1,54 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Status.h" + +#if TD_PORT_WINDOWS +#include "td/utils/port/wstring_convert.h" +#endif + +#if TD_PORT_POSIX +#include "td/utils/port/thread_local.h" + +#include <string.h> + +#include <cstring> +#endif + +namespace td { + +#if TD_PORT_POSIX +CSlice strerror_safe(int code) { + const size_t size = 1000; + + static TD_THREAD_LOCAL char *buf; + init_thread_local<char[]>(buf, size); + +#if !defined(__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) + strerror_r(code, buf, size); + return CSlice(buf, buf + std::strlen(buf)); +#else + return CSlice(strerror_r(code, buf, size)); +#endif +} +#endif + +#if TD_PORT_WINDOWS +string winerror_to_string(int code) { + const size_t size = 1000; + wchar_t wbuf[size]; + auto res_size = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, code, 0, wbuf, size - 1, nullptr); + if (res_size == 0) { + return "Unknown windows error"; + } + while (res_size != 0 && (wbuf[res_size - 1] == '\n' || wbuf[res_size - 1] == '\r')) { + res_size--; + } + return from_wstring(wbuf, res_size).ok(); +} +#endif + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Status.h b/libs/tdlib/td/tdutils/td/utils/Status.h new file mode 100644 index 0000000000..8ef2846df1 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Status.h @@ -0,0 +1,458 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/StringBuilder.h" + +#include <cerrno> +#include <cstring> +#include <new> +#include <utility> + +#define TRY_STATUS(status) \ + { \ + auto try_status = (status); \ + if (try_status.is_error()) { \ + return try_status.move_as_error(); \ + } \ + } +#define TRY_RESULT(name, result) TRY_RESULT_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result) + +#define TRY_RESULT_IMPL(r_name, name, result) \ + auto r_name = (result); \ + if (r_name.is_error()) { \ + return r_name.move_as_error(); \ + } \ + auto name = r_name.move_as_ok(); + +#define LOG_STATUS(status) \ + { \ + auto log_status = (status); \ + if (log_status.is_error()) { \ + LOG(ERROR) << log_status.move_as_error(); \ + } \ + } + +#ifndef TD_STATUS_NO_ENSURE +#define ensure() ensure_impl(__FILE__, __LINE__) +#define ensure_error() ensure_error_impl(__FILE__, __LINE__) +#endif + +#if TD_PORT_POSIX +#define OS_ERROR(message) \ + [&]() { \ + auto saved_errno = errno; \ + return ::td::Status::PosixError(saved_errno, (message)); \ + }() +#define OS_SOCKET_ERROR(message) OS_ERROR(message) +#elif TD_PORT_WINDOWS +#define OS_ERROR(message) \ + [&]() { \ + auto saved_error = ::GetLastError(); \ + return ::td::Status::WindowsError(saved_error, (message)); \ + }() +#define OS_SOCKET_ERROR(message) \ + [&]() { \ + auto saved_error = ::WSAGetLastError(); \ + return ::td::Status::WindowsError(saved_error, (message)); \ + }() +#endif + +namespace td { + +#if TD_PORT_POSIX +CSlice strerror_safe(int code); +#endif + +#if TD_PORT_WINDOWS +string winerror_to_string(int code); +#endif + +class Status { + enum class ErrorType : int8 { general, os }; + + public: + Status() = default; + + bool operator==(const Status &other) const { + if (get_info().static_flag) { + return ptr_ == other.ptr_; + } + return false; + } + + Status clone() const TD_WARN_UNUSED_RESULT { + if (is_ok()) { + return Status(); + } + auto info = get_info(); + if (info.static_flag) { + return clone_static(); + } + return Status(false, info.error_type, info.error_code, message()); + } + + static Status OK() TD_WARN_UNUSED_RESULT { + return Status(); + } + + static Status Error(int err, Slice message = Slice()) TD_WARN_UNUSED_RESULT { + return Status(false, ErrorType::general, err, message); + } + + static Status Error(Slice message) TD_WARN_UNUSED_RESULT { + return Error(0, message); + } + +#if TD_PORT_WINDOWS + static Status WindowsError(int saved_error, Slice message) TD_WARN_UNUSED_RESULT { + return Status(false, ErrorType::os, saved_error, message); + } +#endif + +#if TD_PORT_POSIX + static Status PosixError(int32 saved_errno, Slice message) TD_WARN_UNUSED_RESULT { + return Status(false, ErrorType::os, saved_errno, message); + } +#endif + + static Status Error() TD_WARN_UNUSED_RESULT { + return Error<0>(); + } + + template <int Code> + static Status Error() { + static Status status(true, ErrorType::general, Code, ""); + return status.clone_static(); + } + + static Status InvalidId() TD_WARN_UNUSED_RESULT { + static Status status(true, ErrorType::general, 0, "Invalid Id"); + return status.clone_static(); + } + + static Status Hangup() TD_WARN_UNUSED_RESULT { + static Status status(true, ErrorType::general, 0, "Hangup"); + return status.clone_static(); + } + + StringBuilder &print(StringBuilder &sb) const { + if (is_ok()) { + return sb << "OK"; + } + Info info = get_info(); + switch (info.error_type) { + case ErrorType::general: + sb << "[Error"; + break; + case ErrorType::os: +#if TD_PORT_POSIX + sb << "[PosixError : " << strerror_safe(info.error_code); +#elif TD_PORT_WINDOWS + sb << "[WindowsError : " << winerror_to_string(info.error_code); +#endif + break; + default: + LOG(FATAL) << "Unknown status type: " << static_cast<int8>(info.error_type); + UNREACHABLE(); + break; + } + sb << " : " << code() << " : " << message() << "]"; + return sb; + } + + string to_string() const { + auto buf = StackAllocator::alloc(4096); + StringBuilder sb(buf.as_slice()); + print(sb); + return sb.as_cslice().str(); + } + + // Default interface + bool is_ok() const TD_WARN_UNUSED_RESULT { + return !is_error(); + } + + bool is_error() const TD_WARN_UNUSED_RESULT { + return ptr_ != nullptr; + } + + void ensure_impl(CSlice file_name, int line) const { + if (!is_ok()) { + LOG(FATAL) << "Unexpexted Status " << to_string() << " in file " << file_name << " at line " << line; + } + } + void ensure_error_impl(CSlice file_name, int line) const { + if (is_ok()) { + LOG(FATAL) << "Unexpected Status::OK in file " << file_name << " at line " << line; + } + } + + void ignore() const { + // nop + } + + int32 code() const { + if (is_ok()) { + return 0; + } + return get_info().error_code; + } + + CSlice message() const { + if (is_ok()) { + return CSlice("OK"); + } + return CSlice(ptr_.get() + sizeof(Info)); + } + + string public_message() const { + if (is_ok()) { + return "OK"; + } + Info info = get_info(); + switch (info.error_type) { + case ErrorType::general: + return message().str(); + case ErrorType::os: +#if TD_PORT_POSIX + return strerror_safe(info.error_code).str(); +#elif TD_PORT_WINDOWS + return winerror_to_string(info.error_code); +#endif + default: + LOG(FATAL) << "Unknown status type: " << static_cast<int8>(info.error_type); + UNREACHABLE(); + return ""; + } + } + + const Status &error() const { + return *this; + } + + Status move() TD_WARN_UNUSED_RESULT { + return std::move(*this); + } + + Status move_as_error() TD_WARN_UNUSED_RESULT { + return std::move(*this); + } + + private: + struct Info { + bool static_flag : 1; + signed int error_code : 23; + ErrorType error_type; + }; + + struct Deleter { + void operator()(char *ptr) { + if (!get_info(ptr).static_flag) { + delete[] ptr; + } + } + }; + std::unique_ptr<char[], Deleter> ptr_; + + Status(Info info, Slice message) { + size_t size = sizeof(Info) + message.size() + 1; + ptr_ = std::unique_ptr<char[], Deleter>(new char[size]); + char *ptr = ptr_.get(); + reinterpret_cast<Info *>(ptr)[0] = info; + ptr += sizeof(Info); + std::memcpy(ptr, message.begin(), message.size()); + ptr += message.size(); + *ptr = 0; + } + + Status(bool static_flag, ErrorType error_type, int error_code, Slice message) + : Status(to_info(static_flag, error_type, error_code), message) { + } + + Status clone_static() const TD_WARN_UNUSED_RESULT { + CHECK(is_ok() || get_info().static_flag); + Status result; + result.ptr_ = std::unique_ptr<char[], Deleter>(ptr_.get()); + return result; + } + + static Info to_info(bool static_flag, ErrorType error_type, int error_code) { + const int MIN_ERROR_CODE = -(1 << 22) + 1; + const int MAX_ERROR_CODE = (1 << 22) - 1; + Info tmp; + tmp.static_flag = static_flag; + tmp.error_type = error_type; + + if (error_code < MIN_ERROR_CODE) { + LOG(ERROR) << "Error code value is altered from " << error_code; + error_code = MIN_ERROR_CODE; + } + if (error_code > MAX_ERROR_CODE) { + LOG(ERROR) << "Error code value is altered from " << error_code; + error_code = MAX_ERROR_CODE; + } + +#if TD_GCC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + tmp.error_code = error_code; +#if TD_GCC +#pragma GCC diagnostic pop +#endif + CHECK(error_code == tmp.error_code); + return tmp; + } + + Info get_info() const { + return get_info(ptr_.get()); + } + static Info get_info(char *ptr) { + return reinterpret_cast<Info *>(ptr)[0]; + } +}; + +template <class T = Unit> +class Result { + public: + Result() : status_(Status::Error()) { + } + template <class S> + Result(S &&x) : status_(), value_(std::forward<S>(x)) { + } + Result(Status &&status) : status_(std::move(status)) { + CHECK(status_.is_error()); + } + Result(const Result &) = delete; + Result &operator=(const Result &) = delete; + Result(Result &&other) : status_(std::move(other.status_)) { + if (status_.is_ok()) { + new (&value_) T(std::move(other.value_)); + other.value_.~T(); + } + other.status_ = Status::Error(); + } + Result &operator=(Result &&other) { + if (status_.is_ok()) { + value_.~T(); + } + if (other.status_.is_ok()) { +#if TD_GCC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + new (&value_) T(std::move(other.value_)); +#if TD_GCC +#pragma GCC diagnostic pop +#endif + other.value_.~T(); + } + status_ = std::move(other.status_); + other.status_ = Status::Error(); + return *this; + } + ~Result() { + if (status_.is_ok()) { + value_.~T(); + } + } + + void ensure_impl(CSlice file_name, int line) const { + status_.ensure_impl(file_name, line); + } + void ensure_error_impl(CSlice file_name, int line) const { + status_.ensure_error_impl(file_name, line); + } + void ignore() const { + status_.ignore(); + } + bool is_ok() const { + return status_.is_ok(); + } + bool is_error() const { + return status_.is_error(); + } + const Status &error() const { + CHECK(status_.is_error()); + return status_; + } + Status move_as_error() TD_WARN_UNUSED_RESULT { + CHECK(status_.is_error()); + SCOPE_EXIT { + status_ = Status::Error(); + }; + return std::move(status_); + } + const T &ok() const { + CHECK(status_.is_ok()) << status_; + return value_; + } + T &ok_ref() { + CHECK(status_.is_ok()) << status_; + return value_; + } + T move_as_ok() { + CHECK(status_.is_ok()) << status_; + return std::move(value_); + } + + Result<T> clone() const TD_WARN_UNUSED_RESULT { + if (is_ok()) { + return Result<T>(ok()); // TODO: return clone(ok()); + } + return error().clone(); + } + void clear() { + *this = Result<T>(); + } + + private: + Status status_; + union { + T value_; + }; +}; + +template <> +inline Result<Unit>::Result(Status &&status) : status_(std::move(status)) { + // no assert +} + +inline StringBuilder &operator<<(StringBuilder &string_builder, const Status &status) { + return status.print(string_builder); +} + +namespace detail { + +class SlicifySafe { + public: + Result<CSlice> operator&(Logger &logger) { + if (logger.is_error()) { + return Status::Error("Buffer overflow"); + } + return logger.as_cslice(); + } +}; + +class StringifySafe { + public: + Result<string> operator&(Logger &logger) { + if (logger.is_error()) { + return Status::Error("Buffer overflow"); + } + return logger.as_cslice().str(); + } +}; + +} // namespace detail +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Storer.h b/libs/tdlib/td/tdutils/td/utils/Storer.h new file mode 100644 index 0000000000..91750dcd44 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Storer.h @@ -0,0 +1,86 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once +#include "td/utils/StorerBase.h" + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/tl_storers.h" + +#include <cstring> +#include <limits> + +namespace td { +class SliceStorer : public Storer { + Slice slice; + + public: + explicit SliceStorer(Slice slice) : slice(slice) { + } + size_t size() const override { + return slice.size(); + } + size_t store(uint8 *ptr) const override { + std::memcpy(ptr, slice.ubegin(), slice.size()); + return slice.size(); + } +}; + +inline SliceStorer create_storer(Slice slice) { + return SliceStorer(slice); +} + +class ConcatStorer : public Storer { + const Storer &a_; + const Storer &b_; + + public: + ConcatStorer(const Storer &a, const Storer &b) : a_(a), b_(b) { + } + + size_t size() const override { + return a_.size() + b_.size(); + } + + size_t store(uint8 *ptr) const override { + uint8 *ptr_save = ptr; + ptr += a_.store(ptr); + ptr += b_.store(ptr); + return ptr - ptr_save; + } +}; + +inline ConcatStorer create_storer(const Storer &a, const Storer &b) { + return ConcatStorer(a, b); +} + +template <class T> +class DefaultStorer : public Storer { + public: + explicit DefaultStorer(const T &object) : object_(object) { + } + size_t size() const override { + if (size_ == std::numeric_limits<size_t>::max()) { + size_ = tl_calc_length(object_); + } + return size_; + } + size_t store(uint8 *ptr) const override { + return tl_store_unsafe(object_, ptr); + } + + private: + mutable size_t size_ = std::numeric_limits<size_t>::max(); + const T &object_; +}; + +template <class T> +DefaultStorer<T> create_default_storer(const T &from) { + return DefaultStorer<T>(from); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/StorerBase.h b/libs/tdlib/td/tdutils/td/utils/StorerBase.h new file mode 100644 index 0000000000..e6fea28e16 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/StorerBase.h @@ -0,0 +1,25 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/int_types.h" + +namespace td { + +class Storer { + public: + Storer() = default; + Storer(const Storer &) = delete; + Storer &operator=(const Storer &) = delete; + Storer(Storer &&) = default; + Storer &operator=(Storer &&) = default; + virtual ~Storer() = default; + virtual size_t size() const = 0; + virtual size_t store(uint8 *ptr) const = 0; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/StringBuilder.cpp b/libs/tdlib/td/tdutils/td/utils/StringBuilder.cpp new file mode 100644 index 0000000000..ce64bbc9a6 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/StringBuilder.cpp @@ -0,0 +1,102 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/StringBuilder.h" + +#include "td/utils/misc.h" +#include "td/utils/port/thread_local.h" + +#include <cstdio> +#include <locale> +#include <sstream> + +namespace td { + +// TODO: optimize +StringBuilder &StringBuilder::operator<<(int x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%d", x); + return *this; +} + +StringBuilder &StringBuilder::operator<<(unsigned int x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%u", x); + return *this; +} + +StringBuilder &StringBuilder::operator<<(long int x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%ld", x); + return *this; +} + +StringBuilder &StringBuilder::operator<<(long unsigned int x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%lu", x); + return *this; +} + +StringBuilder &StringBuilder::operator<<(long long int x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%lld", x); + return *this; +} + +StringBuilder &StringBuilder::operator<<(long long unsigned int x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%llu", x); + return *this; +} + +StringBuilder &StringBuilder::operator<<(FixedDouble x) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + + static TD_THREAD_LOCAL std::stringstream *ss; + if (init_thread_local<std::stringstream>(ss)) { + ss->imbue(std::locale::classic()); + ss->setf(std::ios_base::fixed, std::ios_base::floatfield); + } else { + ss->str(std::string()); + ss->clear(); + } + ss->precision(x.precision); + *ss << x.d; + + int len = narrow_cast<int>(static_cast<std::streamoff>(ss->tellp())); + auto left = end_ptr_ + reserved_size - current_ptr_; + if (unlikely(len >= left)) { + error_flag_ = true; + len = left ? narrow_cast<int>(left - 1) : 0; + } + ss->read(current_ptr_, len); + current_ptr_ += len; + return *this; +} + +StringBuilder &StringBuilder::operator<<(const void *ptr) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%p", ptr); + return *this; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/StringBuilder.h b/libs/tdlib/td/tdutils/td/utils/StringBuilder.h new file mode 100644 index 0000000000..a6345a9273 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/StringBuilder.h @@ -0,0 +1,138 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice-decl.h" +#include "td/utils/StackAllocator.h" + +#include <cstdlib> +#include <cstring> +#include <type_traits> + +namespace td { + +class StringBuilder { + public: + explicit StringBuilder(MutableSlice slice) + : begin_ptr_(slice.begin()), current_ptr_(begin_ptr_), end_ptr_(slice.end() - reserved_size) { + if (slice.size() <= reserved_size) { + std::abort(); // shouldn't happen + } + } + + void clear() { + current_ptr_ = begin_ptr_; + error_flag_ = false; + } + + MutableCSlice as_cslice() { + if (current_ptr_ >= end_ptr_ + reserved_size) { + std::abort(); // shouldn't happen + } + *current_ptr_ = 0; + return MutableCSlice(begin_ptr_, current_ptr_); + } + + bool is_error() const { + return error_flag_; + } + + StringBuilder &operator<<(const char *str) { + return *this << Slice(str); + } + + StringBuilder &operator<<(Slice slice) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + auto size = static_cast<size_t>(end_ptr_ + reserved_size - 1 - current_ptr_); + if (unlikely(slice.size() > size)) { + error_flag_ = true; + } else { + size = slice.size(); + } + std::memcpy(current_ptr_, slice.begin(), size); + current_ptr_ += size; + return *this; + } + + StringBuilder &operator<<(bool b) { + return *this << (b ? Slice("true") : Slice("false")); + } + + StringBuilder &operator<<(char c) { + if (unlikely(end_ptr_ < current_ptr_)) { + return on_error(); + } + *current_ptr_++ = c; + return *this; + } + + StringBuilder &operator<<(unsigned char c) { + return *this << static_cast<unsigned int>(c); + } + + StringBuilder &operator<<(signed char c) { + return *this << static_cast<int>(c); + } + + StringBuilder &operator<<(int x); + + StringBuilder &operator<<(unsigned int x); + + StringBuilder &operator<<(long int x); + + StringBuilder &operator<<(long unsigned int x); + + StringBuilder &operator<<(long long int x); + + StringBuilder &operator<<(long long unsigned int x); + + struct FixedDouble { + double d; + int precision; + + FixedDouble(double d, int precision) : d(d), precision(precision) { + } + }; + StringBuilder &operator<<(FixedDouble x); + + StringBuilder &operator<<(double x) { + return *this << FixedDouble(x, 6); + } + + StringBuilder &operator<<(const void *ptr); + + template <class T> + StringBuilder &operator<<(const T *ptr) { + return *this << static_cast<const void *>(ptr); + } + + private: + char *begin_ptr_; + char *current_ptr_; + char *end_ptr_; + bool error_flag_ = false; + static constexpr size_t reserved_size = 30; + + StringBuilder &on_error() { + error_flag_ = true; + return *this; + } +}; + +template <class T> +std::enable_if_t<std::is_arithmetic<T>::value, string> to_string(const T &x) { + const size_t buf_size = 1000; + auto buf = StackAllocator::alloc(buf_size); + StringBuilder sb(buf.as_slice()); + sb << x; + return sb.as_cslice().str(); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Time.cpp b/libs/tdlib/td/tdutils/td/utils/Time.cpp new file mode 100644 index 0000000000..3e62002c18 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Time.cpp @@ -0,0 +1,19 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Time.h" + +#include <cmath> + +namespace td { + +std::atomic<double> Time::now_; + +bool operator==(Timestamp a, Timestamp b) { + return std::abs(a.at() - b.at()) < 1e-6; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Time.h b/libs/tdlib/td/tdutils/td/utils/Time.h new file mode 100644 index 0000000000..acdb8b52ef --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Time.h @@ -0,0 +1,104 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/Clocks.h" + +#include <atomic> + +namespace td { + +class Time { + public: + static double now() { + double now = Clocks::monotonic(); + now_.store(now, std::memory_order_relaxed); + return now; + } + static double now_cached() { + return now_.load(std::memory_order_relaxed); + } + + private: + static std::atomic<double> now_; +}; + +inline void relax_timeout_at(double *timeout, double new_timeout) { + if (new_timeout == 0) { + return; + } + if (*timeout == 0 || new_timeout < *timeout) { + *timeout = new_timeout; + } +} + +class Timestamp { + public: + Timestamp() = default; + static Timestamp never() { + return Timestamp{}; + } + static Timestamp now() { + return Timestamp{Time::now()}; + } + static Timestamp now_cached() { + return Timestamp{Time::now_cached()}; + } + static Timestamp at(double timeout) { + return Timestamp{timeout}; + } + + static Timestamp in(double timeout) { + return Timestamp{Time::now_cached() + timeout}; + } + + bool is_in_past() const { + return at_ <= Time::now_cached(); + } + + explicit operator bool() const { + return at_ > 0; + } + + double at() const { + return at_; + } + + double in() const { + return at_ - Time::now_cached(); + } + + void relax(const Timestamp &timeout) { + if (!timeout) { + return; + } + if (!*this || at_ > timeout.at_) { + at_ = timeout.at_; + } + } + + friend bool operator==(Timestamp a, Timestamp b); + + private: + double at_{0}; + + explicit Timestamp(double timeout) : at_(timeout) { + } +}; + +template <class T> +void parse(Timestamp ×tamp, T &parser) { + timestamp = Timestamp::in(parser.fetch_double() - Clocks::system()); +} + +template <class T> +void store(const Timestamp ×tamp, T &storer) { + storer.store_binary(timestamp.at() - Time::now() + Clocks::system()); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/TimedStat.h b/libs/tdlib/td/tdutils/td/utils/TimedStat.h new file mode 100644 index 0000000000..fc4197470d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/TimedStat.h @@ -0,0 +1,71 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/logging.h" + +#include <utility> + +namespace td { + +template <class StatT> +class TimedStat { + public: + TimedStat(double duration, double now) + : duration_(duration), current_(), current_timestamp_(now), next_(), next_timestamp_(now) { + } + TimedStat() : TimedStat(0, 0) { + } + template <class EventT> + void add_event(const EventT &e, double now) { + update(now); + current_.on_event(e); + next_.on_event(e); + } + const StatT &get_stat(double now) { + update(now); + return current_; + } + std::pair<StatT, double> stat_duration(double now) { + update(now); + return std::make_pair(current_, now - current_timestamp_); + } + void clear_events() { + current_.clear(); + next_.clear(); + } + + private: + double duration_; + StatT current_; + double current_timestamp_; + StatT next_; + double next_timestamp_; + + void update(double &now) { + if (now < next_timestamp_) { + CHECK(now >= next_timestamp_ * (1 - 1e-14)) << now << " " << next_timestamp_; + now = next_timestamp_; + } + if (duration_ == 0) { + return; + } + if (next_timestamp_ + 2 * duration_ < now) { + current_ = StatT(); + current_timestamp_ = now; + next_ = StatT(); + next_timestamp_ = now; + } else if (next_timestamp_ + duration_ < now) { + current_ = next_; + current_timestamp_ = next_timestamp_; + next_ = StatT(); + next_timestamp_ = now; + } + } +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Timer.cpp b/libs/tdlib/td/tdutils/td/utils/Timer.cpp new file mode 100644 index 0000000000..dc35721caa --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Timer.cpp @@ -0,0 +1,41 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Timer.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +//#include "td/utils/Slice.h" // TODO move StringBuilder implementation to cpp, remove header +#include "td/utils/Time.h" + +namespace td { + +Timer::Timer() : start_time_(Time::now()) { +} + +StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) { + return string_builder << "in " << Time::now() - timer.start_time_; +} + +PerfWarningTimer::PerfWarningTimer(string name, double max_duration) + : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration) { +} + +PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other) + : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_) { + other.start_at_ = 0; +} + +PerfWarningTimer::~PerfWarningTimer() { + if (start_at_ == 0) { + return; + } + double duration = Time::now() - start_at_; + LOG_IF(WARNING, duration > max_duration_) + << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Timer.h b/libs/tdlib/td/tdutils/td/utils/Timer.h new file mode 100644 index 0000000000..65b879088d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Timer.h @@ -0,0 +1,38 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/StringBuilder.h" + +namespace td { + +class Timer { + public: + Timer(); + + private: + friend StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer); + + double start_time_; +}; + +class PerfWarningTimer { + public: + explicit PerfWarningTimer(string name, double max_duration = 0.1); + PerfWarningTimer(const PerfWarningTimer &) = delete; + PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; + PerfWarningTimer(PerfWarningTimer &&other); + PerfWarningTimer &operator=(PerfWarningTimer &&) = delete; + ~PerfWarningTimer(); + + private: + string name_; + double start_at_{0}; + double max_duration_{0}; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/Variant.h b/libs/tdlib/td/tdutils/td/utils/Variant.h new file mode 100644 index 0000000000..9b6e0561cc --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/Variant.h @@ -0,0 +1,286 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" + +#include <new> +#include <type_traits> +#include <utility> + +namespace td { +namespace detail { + +template <size_t... Args> +class MaxSizeImpl {}; + +template <class T> +constexpr const T &constexpr_max(const T &a, const T &b) { + return a < b ? b : a; +} + +template <size_t Res, size_t X, size_t... Args> +class MaxSizeImpl<Res, X, Args...> { + public: + static constexpr size_t value = MaxSizeImpl<constexpr_max(Res, X), Args...>::value; +}; + +template <size_t Res> +class MaxSizeImpl<Res> { + public: + static constexpr size_t value = Res; +}; + +template <class... Args> +class MaxSize { + public: + static constexpr size_t value = MaxSizeImpl<0, sizeof(Args)...>::value; +}; + +template <size_t to_skip, class... Args> +class IthTypeImpl {}; +template <class Res, class... Args> +class IthTypeImpl<0, Res, Args...> { + public: + using type = Res; +}; +template <size_t pos, class Skip, class... Args> +class IthTypeImpl<pos, Skip, Args...> : public IthTypeImpl<pos - 1, Args...> {}; + +class Dummy {}; + +template <size_t pos, class... Args> +class IthType : public IthTypeImpl<pos, Args..., Dummy> {}; + +template <bool ok, int offset, class... Types> +class FindTypeOffsetImpl {}; + +template <int offset, class... Types> +class FindTypeOffsetImpl<true, offset, Types...> { + public: + static constexpr int value = offset; +}; +template <int offset, class T, class S, class... Types> +class FindTypeOffsetImpl<false, offset, T, S, Types...> + : public FindTypeOffsetImpl<std::is_same<T, S>::value, offset + 1, T, Types...> {}; +template <class T, class... Types> +class FindTypeOffset : public FindTypeOffsetImpl<false, -1, T, Types...> {}; + +template <int offset, class... Types> +class ForEachTypeImpl {}; + +template <int offset> +class ForEachTypeImpl<offset, Dummy> { + public: + template <class F> + static void visit(F &&f) { + } +}; + +template <int offset, class T, class... Types> +class ForEachTypeImpl<offset, T, Types...> { + public: + template <class F> + static void visit(F &&f) { + f(offset, static_cast<T *>(nullptr)); + ForEachTypeImpl<offset + 1, Types...>::visit(f); + } +}; + +template <class... Types> +class ForEachType { + public: + template <class F> + static void visit(F &&f) { + ForEachTypeImpl<0, Types..., Dummy>::visit(f); + } +}; + +} // namespace detail + +template <class... Types> +class Variant { + public: + static constexpr int npos = -1; + Variant() { + } + Variant(Variant &&other) { + other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); }); + } + Variant(const Variant &other) { + other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); }); + } + Variant &operator=(Variant &&other) { + clear(); + other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); }); + return *this; + } + Variant &operator=(const Variant &other) { + clear(); + other.visit([&](auto &&value) { this->init_empty(std::forward<decltype(value)>(value)); }); + return *this; + } + + bool operator==(const Variant &other) const { + if (offset_ != other.offset_) { + return false; + } + bool res = false; + for_each([&](int offset, auto *ptr) { + using T = std::decay_t<decltype(*ptr)>; + if (offset == offset_) { + res = this->get<T>() == other.template get<T>(); + } + }); + return res; + } + bool operator<(const Variant &other) const { + if (offset_ != other.offset_) { + return offset_ < other.offset_; + } + bool res = false; + for_each([&](int offset, auto *ptr) { + using T = std::decay_t<decltype(*ptr)>; + if (offset == offset_) { + res = this->get<T>() < other.template get<T>(); + } + }); + return res; + } + + template <class T> + Variant(T &&t) { + init_empty(std::forward<T>(t)); + } + template <class T> + Variant &operator=(T &&t) { + clear(); + init_empty(std::forward<T>(t)); + return *this; + } + template <class T> + static constexpr int offset() { + return detail::FindTypeOffset<std::decay_t<T>, Types...>::value; + } + + template <class T> + void init_empty(T &&t) { + CHECK(offset_ == npos); + offset_ = offset<T>(); + new (&get<T>()) std::decay_t<T>(std::forward<T>(t)); + } + ~Variant() { + clear(); + } + + template <class F> + void visit(F &&f) { + for_each([&](int offset, auto *ptr) { + using T = std::decay_t<decltype(*ptr)>; + if (offset == offset_) { + f(std::move(*this->get_unsafe<T>())); + } + }); + } + template <class F> + void for_each(F &&f) { + detail::ForEachType<Types...>::visit(f); + } + template <class F> + void visit(F &&f) const { + for_each([&](int offset, auto *ptr) { + using T = std::decay_t<decltype(*ptr)>; + if (offset == offset_) { + f(std::move(*this->get_unsafe<T>())); + } + }); + } + template <class F> + void for_each(F &&f) const { + detail::ForEachType<Types...>::visit(f); + } + + void clear() { + visit([](auto &&value) { + using T = std::decay_t<decltype(value)>; + value.~T(); + }); + offset_ = npos; + } + + template <int offset> + auto &get() { + CHECK(offset == offset_); + return *get_unsafe<offset>(); + } + template <class T> + auto &get() { + return get<offset<T>()>(); + } + + template <int offset> + const auto &get() const { + CHECK(offset == offset_); + return *get_unsafe<offset>(); + } + template <class T> + const auto &get() const { + return get<offset<T>()>(); + } + + int32 get_offset() const { + return offset_; + } + + private: + union { + int64 align_; + char data_[detail::MaxSize<Types...>::value]; + }; + int offset_{npos}; + + template <class T> + auto *get_unsafe() { + return reinterpret_cast<T *>(data_); + } + + template <int offset> + auto *get_unsafe() { + using T = typename detail::IthType<offset, Types...>::type; + return get_unsafe<T>(); + } + + template <class T> + const auto *get_unsafe() const { + return reinterpret_cast<const T *>(data_); + } + + template <int offset> + const auto *get_unsafe() const { + using T = typename detail::IthType<offset, Types...>::type; + return get_unsafe<T>(); + } +}; + +template <class T, class... Types> +auto &get(Variant<Types...> &v) { + return v.template get<T>(); +} +template <class T, class... Types> +auto &get(const Variant<Types...> &v) { + return v.template get<T>(); +} +template <int T, class... Types> +auto &get(Variant<Types...> &v) { + return v.template get<T>(); +} +template <int T, class... Types> +auto &get(const Variant<Types...> &v) { + return v.template get<T>(); +} +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/base64.cpp b/libs/tdlib/td/tdutils/td/utils/base64.cpp new file mode 100644 index 0000000000..4016feaa58 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/base64.cpp @@ -0,0 +1,261 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/base64.h" + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <algorithm> +#include <iterator> + +namespace td { +//TODO: fix copypaste + +static const char *const symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +string base64_encode(Slice input) { + string base64; + base64.reserve((input.size() + 2) / 3 * 4); + for (size_t i = 0; i < input.size();) { + size_t left = min(input.size() - i, static_cast<size_t>(3)); + int c = input.ubegin()[i++] << 16; + base64 += symbols64[c >> 18]; + if (left != 1) { + c |= input.ubegin()[i++] << 8; + } + base64 += symbols64[(c >> 12) & 63]; + if (left == 3) { + c |= input.ubegin()[i++]; + } + if (left != 1) { + base64 += symbols64[(c >> 6) & 63]; + } else { + base64 += '='; + } + if (left == 3) { + base64 += symbols64[c & 63]; + } else { + base64 += '='; + } + } + return base64; +} + +static unsigned char char_to_value[256]; +static void init_base64_table() { + static bool is_inited = []() { + std::fill(std::begin(char_to_value), std::end(char_to_value), 64); + for (unsigned char i = 0; i < 64; i++) { + char_to_value[static_cast<size_t>(symbols64[i])] = i; + } + return true; + }(); + CHECK(is_inited); +} + +Result<string> base64_decode(Slice base64) { + init_base64_table(); + + if ((base64.size() & 3) != 0) { + return Status::Error("Wrong string length"); + } + + size_t padding_length = 0; + while (!base64.empty() && base64.back() == '=') { + base64.remove_suffix(1); + padding_length++; + } + if (padding_length >= 3) { + return Status::Error("Wrong string padding"); + } + + string output; + output.reserve(((base64.size() + 3) >> 2) * 3); + for (size_t i = 0; i < base64.size();) { + size_t left = min(base64.size() - i, static_cast<size_t>(4)); + int c = 0; + for (size_t t = 0; t < left; t++) { + auto value = char_to_value[base64.ubegin()[i++]]; + if (value == 64) { + return Status::Error("Wrong character in the string"); + } + c |= value << ((3 - t) * 6); + } + output += static_cast<char>(static_cast<unsigned char>(c >> 16)); // implementation-defined + if (left == 2) { + if ((c & ((1 << 16) - 1)) != 0) { + return Status::Error("Wrong padding in the string"); + } + } else { + output += static_cast<char>(static_cast<unsigned char>(c >> 8)); // implementation-defined + if (left == 3) { + if ((c & ((1 << 8) - 1)) != 0) { + return Status::Error("Wrong padding in the string"); + } + } else { + output += static_cast<char>(static_cast<unsigned char>(c)); // implementation-defined + } + } + } + return output; +} + +static const char *const url_symbols64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +string base64url_encode(Slice input) { + string base64; + base64.reserve((input.size() + 2) / 3 * 4); + for (size_t i = 0; i < input.size();) { + size_t left = min(input.size() - i, static_cast<size_t>(3)); + int c = input.ubegin()[i++] << 16; + base64 += url_symbols64[c >> 18]; + if (left != 1) { + c |= input.ubegin()[i++] << 8; + } + base64 += url_symbols64[(c >> 12) & 63]; + if (left == 3) { + c |= input.ubegin()[i++]; + } + if (left != 1) { + base64 += url_symbols64[(c >> 6) & 63]; + } + if (left == 3) { + base64 += url_symbols64[c & 63]; + } + } + return base64; +} + +static unsigned char url_char_to_value[256]; +static void init_base64url_table() { + static bool is_inited = []() { + std::fill(std::begin(url_char_to_value), std::end(url_char_to_value), 64); + for (unsigned char i = 0; i < 64; i++) { + url_char_to_value[static_cast<size_t>(url_symbols64[i])] = i; + } + return true; + }(); + CHECK(is_inited); +} + +Result<string> base64url_decode(Slice base64) { + init_base64url_table(); + + size_t padding_length = 0; + while (!base64.empty() && base64.back() == '=') { + base64.remove_suffix(1); + padding_length++; + } + if (padding_length >= 3 || (padding_length > 0 && ((base64.size() + padding_length) & 3) != 0)) { + return Status::Error("Wrong string padding"); + } + + if ((base64.size() & 3) == 1) { + return Status::Error("Wrong string length"); + } + + string output; + output.reserve(((base64.size() + 3) >> 2) * 3); + for (size_t i = 0; i < base64.size();) { + size_t left = min(base64.size() - i, static_cast<size_t>(4)); + int c = 0; + for (size_t t = 0; t < left; t++) { + auto value = url_char_to_value[base64.ubegin()[i++]]; + if (value == 64) { + return Status::Error("Wrong character in the string"); + } + c |= value << ((3 - t) * 6); + } + output += static_cast<char>(static_cast<unsigned char>(c >> 16)); // implementation-defined + if (left == 2) { + if ((c & ((1 << 16) - 1)) != 0) { + return Status::Error("Wrong padding in the string"); + } + } else { + output += static_cast<char>(static_cast<unsigned char>(c >> 8)); // implementation-defined + if (left == 3) { + if ((c & ((1 << 8) - 1)) != 0) { + return Status::Error("Wrong padding in the string"); + } + } else { + output += static_cast<char>(static_cast<unsigned char>(c)); // implementation-defined + } + } + } + return output; +} + +template <bool is_url> +static bool is_base64_impl(Slice input) { + size_t padding_length = 0; + while (!input.empty() && input.back() == '=') { + input.remove_suffix(1); + padding_length++; + } + if (padding_length >= 3) { + return false; + } + if ((!is_url || padding_length > 0) && ((input.size() + padding_length) & 3) != 0) { + return false; + } + if (is_url && (input.size() & 3) == 1) { + return false; + } + + unsigned char *table; + if (is_url) { + init_base64url_table(); + table = url_char_to_value; + } else { + init_base64_table(); + table = char_to_value; + } + for (auto c : input) { + if (table[static_cast<unsigned char>(c)] == 64) { + return false; + } + } + + if ((input.size() & 3) == 2) { + auto value = table[static_cast<int>(input.back())]; + if ((value & 15) != 0) { + return false; + } + } + if ((input.size() & 3) == 3) { + auto value = table[static_cast<int>(input.back())]; + if ((value & 3) != 0) { + return false; + } + } + + return true; +} + +bool is_base64(Slice input) { + return is_base64_impl<false>(input); +} + +bool is_base64url(Slice input) { + return is_base64_impl<true>(input); +} + +string base64_filter(Slice input) { + string res; + res.reserve(input.size()); + init_base64_table(); + for (auto c : input) { + if (char_to_value[static_cast<unsigned char>(c)] != 64 || c == '=') { + res += c; + } + } + return res; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/base64.h b/libs/tdlib/td/tdutils/td/utils/base64.h new file mode 100644 index 0000000000..cef2b4cb34 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/base64.h @@ -0,0 +1,26 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +string base64_encode(Slice input); +Result<string> base64_decode(Slice base64); + +string base64url_encode(Slice input); +Result<string> base64url_decode(Slice base64); + +bool is_base64(Slice input); +bool is_base64url(Slice input); + +string base64_filter(Slice input); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/benchmark.h b/libs/tdlib/td/tdutils/td/utils/benchmark.h new file mode 100644 index 0000000000..ddc7ad75e6 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/benchmark.h @@ -0,0 +1,132 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/port/Clocks.h" +#include "td/utils/StringBuilder.h" + +#include <cmath> +#include <tuple> +#include <utility> + +#define BENCH(name, desc) \ + class name##Bench : public ::td::Benchmark { \ + public: \ + std::string get_description() const override { \ + return (desc); \ + } \ + void run(int n) override; \ + }; \ + void name##Bench::run(int n) + +namespace td { + +#if TD_MSVC + +#pragma optimize("", off) +template <class T> +void do_not_optimize_away(T &&datum) { + datum = datum; +} +#pragma optimize("", on) + +#else + +template <class T> +void do_not_optimize_away(T &&datum) { + asm volatile("" : "+r"(datum)); +} + +#endif + +class Benchmark { + public: + Benchmark() = default; + Benchmark(const Benchmark &) = delete; + Benchmark &operator=(const Benchmark &) = delete; + Benchmark(Benchmark &&) = delete; + Benchmark &operator=(Benchmark &&) = delete; + virtual ~Benchmark() = default; + + virtual std::string get_description() const = 0; + + virtual void start_up() { + } + virtual void start_up_n(int n) { + start_up(); + } + + virtual void tear_down() { + } + + virtual void run(int n) = 0; +}; + +inline std::pair<double, double> bench_n(Benchmark &b, int n) { + double total = -Clocks::monotonic(); + b.start_up_n(n); + double t = -Clocks::monotonic(); + b.run(n); + t += Clocks::monotonic(); + b.tear_down(); + total += Clocks::monotonic(); + + return std::make_pair(t, total); +} + +inline std::pair<double, double> bench_n(Benchmark &&b, int n) { + return bench_n(b, n); +} + +inline void bench(Benchmark &b, double max_time = 1.0) { + int n = 1; + double pass_time = 0; + double total_pass_time = 0; + while (pass_time < max_time && total_pass_time < max_time * 3 && n < (1 << 30)) { + n *= 2; + std::tie(pass_time, total_pass_time) = bench_n(b, n); + } + pass_time = n / pass_time; + + int pass_cnt = 2; + double sum = pass_time; + double square_sum = pass_time * pass_time; + double min_pass_time = pass_time; + double max_pass_time = pass_time; + + for (int i = 1; i < pass_cnt; i++) { + pass_time = n / bench_n(b, n).first; + sum += pass_time; + square_sum += pass_time * pass_time; + if (pass_time < min_pass_time) { + min_pass_time = pass_time; + } + if (pass_time > max_pass_time) { + max_pass_time = pass_time; + } + } + double average = sum / pass_cnt; + double d = sqrt(square_sum / pass_cnt - average * average); + + auto description = b.get_description(); + std::string pad; + if (description.size() < 40) { + pad = std::string(40 - description.size(), ' '); + } + + LOG(ERROR) << "Bench [" << pad << description << "]: " << StringBuilder::FixedDouble(average, 3) << '[' + << StringBuilder::FixedDouble(min_pass_time, 3) << '-' << StringBuilder::FixedDouble(max_pass_time, 3) + << "] ops/sec,\t" << format::as_time(1 / average) << " [d = " << StringBuilder::FixedDouble(d, 6) << ']'; +} + +inline void bench(Benchmark &&b, double max_time = 1.0) { + bench(b, max_time); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/buffer.cpp b/libs/tdlib/td/tdutils/td/utils/buffer.cpp new file mode 100644 index 0000000000..c1a123031c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/buffer.cpp @@ -0,0 +1,105 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/buffer.h" + +#include "td/utils/port/thread_local.h" + +#include <new> + +namespace td { + +TD_THREAD_LOCAL BufferAllocator::BufferRawTls *BufferAllocator::buffer_raw_tls; // static zero-initialized + +std::atomic<size_t> BufferAllocator::buffer_mem; + +size_t BufferAllocator::get_buffer_mem() { + return buffer_mem; +} + +BufferAllocator::WriterPtr BufferAllocator::create_writer(size_t size) { + if (size < 512) { + size = 512; + } + return create_writer_exact(size); +} + +BufferAllocator::WriterPtr BufferAllocator::create_writer_exact(size_t size) { + return WriterPtr(create_buffer_raw(size)); +} + +BufferAllocator::WriterPtr BufferAllocator::create_writer(size_t size, size_t prepend, size_t append) { + auto ptr = create_writer(size + prepend + append); + ptr->begin_ += prepend; + ptr->end_ += prepend + size; + return ptr; +} + +BufferAllocator::ReaderPtr BufferAllocator::create_reader(size_t size) { + if (size < 512) { + return create_reader_fast(size); + } + auto ptr = create_writer_exact(size); + ptr->end_ += (size + 7) & -8; + return create_reader(ptr); +} + +BufferAllocator::ReaderPtr BufferAllocator::create_reader_fast(size_t size) { + size = (size + 7) & -8; + + init_thread_local<BufferRawTls>(buffer_raw_tls); + + auto buffer_raw = buffer_raw_tls->buffer_raw.get(); + if (buffer_raw == nullptr || buffer_raw->data_size_ - buffer_raw->end_.load(std::memory_order_relaxed) < size) { + buffer_raw = create_buffer_raw(4096 * 4); + buffer_raw_tls->buffer_raw = std::unique_ptr<BufferRaw, BufferAllocator::BufferRawDeleter>(buffer_raw); + } + buffer_raw->end_.fetch_add(size, std::memory_order_relaxed); + buffer_raw->ref_cnt_.fetch_add(1, std::memory_order_acq_rel); + return ReaderPtr(buffer_raw); +} + +BufferAllocator::ReaderPtr BufferAllocator::create_reader(const WriterPtr &raw) { + raw->was_reader_ = true; + raw->ref_cnt_.fetch_add(1, std::memory_order_acq_rel); + return ReaderPtr(raw.get()); +} + +BufferAllocator::ReaderPtr BufferAllocator::create_reader(const ReaderPtr &raw) { + raw->ref_cnt_.fetch_add(1, std::memory_order_acq_rel); + return ReaderPtr(raw.get()); +} + +void BufferAllocator::dec_ref_cnt(BufferRaw *ptr) { + int left = ptr->ref_cnt_.fetch_sub(1, std::memory_order_acq_rel); + if (left == 1) { + auto buf_size = max(sizeof(BufferRaw), offsetof(BufferRaw, data_) + ptr->data_size_); + buffer_mem -= buf_size; + ptr->~BufferRaw(); + delete[] ptr; + } +} + +BufferRaw *BufferAllocator::create_buffer_raw(size_t size) { + size = (size + 7) & -8; + + auto buf_size = offsetof(BufferRaw, data_) + size; + if (buf_size < sizeof(BufferRaw)) { + buf_size = sizeof(BufferRaw); + } + buffer_mem += buf_size; + auto *buffer_raw = reinterpret_cast<BufferRaw *>(new char[buf_size]); + new (buffer_raw) BufferRaw(); + buffer_raw->data_size_ = size; + buffer_raw->begin_ = 0; + buffer_raw->end_ = 0; + + buffer_raw->ref_cnt_.store(1, std::memory_order_relaxed); + buffer_raw->has_writer_.store(true, std::memory_order_relaxed); + buffer_raw->was_reader_ = false; + return buffer_raw; +} +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/buffer.h b/libs/tdlib/td/tdutils/td/utils/buffer.h new file mode 100644 index 0000000000..aa4ef8db26 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/buffer.h @@ -0,0 +1,708 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/Slice.h" + +#include <atomic> +#include <cstring> +#include <limits> + +namespace td { + +struct BufferRaw { + size_t data_size_; + + // Constant after first reader is created. + // May be change by writer before it. + // So writer may do prepends till there is no reader created. + size_t begin_; + + // Write by writer. + // Read by reader. + std::atomic<size_t> end_; + + mutable std::atomic<int32> ref_cnt_; + std::atomic<bool> has_writer_; + bool was_reader_; + + alignas(4) char data_[1]; +}; + +class BufferAllocator { + public: + class DeleteWriterPtr { + public: + void operator()(BufferRaw *ptr) { + ptr->has_writer_.store(false, std::memory_order_release); + dec_ref_cnt(ptr); + } + }; + class DeleteReaderPtr { + public: + void operator()(BufferRaw *ptr) { + dec_ref_cnt(ptr); + } + }; + + using WriterPtr = std::unique_ptr<BufferRaw, DeleteWriterPtr>; + using ReaderPtr = std::unique_ptr<BufferRaw, DeleteReaderPtr>; + + static WriterPtr create_writer(size_t size); + + static WriterPtr create_writer(size_t size, size_t prepend, size_t append); + + static ReaderPtr create_reader(size_t size); + + static ReaderPtr create_reader(const WriterPtr &raw); + + static ReaderPtr create_reader(const ReaderPtr &raw); + + static size_t get_buffer_mem(); + + static void clear_thread_local(); + + private: + static ReaderPtr create_reader_fast(size_t size); + + static WriterPtr create_writer_exact(size_t size); + + struct BufferRawDeleter { + void operator()(BufferRaw *ptr) { + dec_ref_cnt(ptr); + } + }; + struct BufferRawTls { + std::unique_ptr<BufferRaw, BufferRawDeleter> buffer_raw; + }; + + static TD_THREAD_LOCAL BufferRawTls *buffer_raw_tls; + + static void dec_ref_cnt(BufferRaw *ptr); + + static BufferRaw *create_buffer_raw(size_t size); + + static std::atomic<size_t> buffer_mem; +}; + +using BufferWriterPtr = BufferAllocator::WriterPtr; +using BufferReaderPtr = BufferAllocator::ReaderPtr; + +class BufferSlice { + public: + BufferSlice() = default; + explicit BufferSlice(BufferReaderPtr buffer_ptr) : buffer_(std::move(buffer_ptr)) { + if (is_null()) { + return; + } + begin_ = buffer_->begin_; + sync_with_writer(); + } + BufferSlice(BufferReaderPtr buffer_ptr, size_t begin, size_t end) + : buffer_(std::move(buffer_ptr)), begin_(begin), end_(end) { + } + + explicit BufferSlice(size_t size) : buffer_(BufferAllocator::create_reader(size)) { + end_ = buffer_->end_.load(std::memory_order_relaxed); + begin_ = end_ - ((size + 7) & -8); + end_ = begin_ + size; + } + + explicit BufferSlice(Slice slice) : BufferSlice(slice.size()) { + std::memcpy(as_slice().begin(), slice.begin(), slice.size()); + } + + BufferSlice(const char *ptr, size_t size) : BufferSlice(Slice(ptr, size)) { + } + + BufferSlice clone() const { + if (is_null()) { + return BufferSlice(BufferReaderPtr(), begin_, end_); + } + return BufferSlice(BufferAllocator::create_reader(buffer_), begin_, end_); + } + + BufferSlice copy() const { + if (is_null()) { + return BufferSlice(BufferReaderPtr(), begin_, end_); + } + return BufferSlice(as_slice()); + } + + Slice as_slice() const { + if (is_null()) { + return Slice(); + } + return Slice(buffer_->data_ + begin_, size()); + } + + MutableSlice as_slice() { + if (is_null()) { + return MutableSlice(); + } + return MutableSlice(buffer_->data_ + begin_, size()); + } + + Slice prepare_read() const { + return as_slice(); + } + + Slice after(size_t offset) const { + auto full = as_slice(); + full.remove_prefix(offset); + return full; + } + + bool confirm_read(size_t size) { + begin_ += size; + CHECK(begin_ <= end_); + return begin_ == end_; + } + + void truncate(size_t limit) { + if (size() > limit) { + end_ = begin_ + limit; + } + } + + BufferSlice from_slice(Slice slice) const { + auto res = BufferSlice(BufferAllocator::create_reader(buffer_)); + res.begin_ = slice.begin() - buffer_->data_; + res.end_ = slice.end() - buffer_->data_; + CHECK(buffer_->begin_ <= res.begin_); + CHECK(res.begin_ <= res.end_); + CHECK(res.end_ <= buffer_->end_.load(std::memory_order_relaxed)); + return res; + } + + // like in std::string + char *data() { + return as_slice().data(); + } + const char *data() const { + return as_slice().data(); + } + char operator[](size_t at) const { + return as_slice()[at]; + } + + bool empty() const { + return size() == 0; + } + + bool is_null() const { + return !buffer_; + } + + size_t size() const { + return end_ - begin_; + } + + // set end_ into writer's end_ + size_t sync_with_writer() { + CHECK(!is_null()); + auto old_end = end_; + end_ = buffer_->end_.load(std::memory_order_acquire); + return end_ - old_end; + } + bool is_writer_alive() const { + CHECK(!is_null()); + return buffer_->has_writer_.load(std::memory_order_acquire); + } + + private: + BufferReaderPtr buffer_; + size_t begin_ = 0; + size_t end_ = 0; +}; + +template <class StorerT> +void store(const BufferSlice &buffer_slice, StorerT &storer) { + storer.store_string(buffer_slice); +} + +template <class ParserT> +void parse(BufferSlice &buffer_slice, ParserT &parser) { + buffer_slice = parser.template fetch_string<BufferSlice>(); +} + +class BufferWriter { + public: + BufferWriter() = default; + explicit BufferWriter(size_t size) : BufferWriter(BufferAllocator::create_writer(size)) { + } + BufferWriter(size_t size, size_t prepend, size_t append) + : BufferWriter(BufferAllocator::create_writer(size, prepend, append)) { + } + explicit BufferWriter(BufferWriterPtr buffer_ptr) : buffer_(std::move(buffer_ptr)) { + } + + BufferSlice as_buffer_slice() const { + return BufferSlice(BufferAllocator::create_reader(buffer_)); + } + bool is_null() const { + return !buffer_; + } + bool empty() const { + return size() == 0; + } + size_t size() const { + if (is_null()) { + return 0; + } + return buffer_->end_.load(std::memory_order_relaxed) - buffer_->begin_; + } + MutableSlice as_slice() { + auto end = buffer_->end_.load(std::memory_order_relaxed); + return MutableSlice(buffer_->data_ + buffer_->begin_, buffer_->data_ + end); + } + + MutableSlice prepare_prepend() { + if (is_null()) { + return MutableSlice(); + } + CHECK(!buffer_->was_reader_); + return MutableSlice(buffer_->data_, buffer_->begin_); + } + MutableSlice prepare_append() { + if (is_null()) { + return MutableSlice(); + } + auto end = buffer_->end_.load(std::memory_order_relaxed); + return MutableSlice(buffer_->data_ + end, buffer_->data_size_ - end); + } + void confirm_append(size_t size) { + if (is_null()) { + CHECK(size == 0); + return; + } + auto new_end = buffer_->end_.load(std::memory_order_relaxed) + size; + CHECK(new_end <= buffer_->data_size_); + buffer_->end_.store(new_end, std::memory_order_release); + } + void confirm_prepend(size_t size) { + if (is_null()) { + CHECK(size == 0); + return; + } + CHECK(buffer_->begin_ >= size); + buffer_->begin_ -= size; + } + + private: + BufferWriterPtr buffer_; +}; + +struct ChainBufferNode { + friend struct DeleteWriterPtr; + struct DeleteWriterPtr { + void operator()(ChainBufferNode *ptr) { + ptr->has_writer_.store(false, std::memory_order_release); + dec_ref_cnt(ptr); + } + }; + friend struct DeleteReaderPtr; + struct DeleteReaderPtr { + void operator()(ChainBufferNode *ptr) { + dec_ref_cnt(ptr); + } + }; + using WriterPtr = std::unique_ptr<ChainBufferNode, DeleteWriterPtr>; + using ReaderPtr = std::unique_ptr<ChainBufferNode, DeleteReaderPtr>; + + static WriterPtr make_writer_ptr(ChainBufferNode *ptr) { + ptr->ref_cnt_.store(1, std::memory_order_relaxed); + ptr->has_writer_.store(true, std::memory_order_relaxed); + return WriterPtr(ptr); + } + static ReaderPtr make_reader_ptr(ChainBufferNode *ptr) { + ptr->ref_cnt_.fetch_add(1, std::memory_order_acq_rel); + return ReaderPtr(ptr); + } + + bool has_writer() { + return has_writer_.load(std::memory_order_acquire); + } + + bool unique() { + return ref_cnt_.load(std::memory_order_acquire) == 1; + } + + ChainBufferNode(BufferSlice slice, bool sync_flag) : slice_(std::move(slice)), sync_flag_(sync_flag) { + } + + // reader + // There are two options + // 1. Fixed slice of Buffer + // 2. Slice with non-fixed right end + // In each case slice_ is const. Reader should read it and use sync_with_writer on its own copy. + const BufferSlice slice_; + const bool sync_flag_{false}; // should we call slice_.sync_with_writer or not. + + // writer + ReaderPtr next_{nullptr}; + + private: + std::atomic<int> ref_cnt_{0}; + std::atomic<bool> has_writer_{false}; + + static void clear_nonrecursive(ReaderPtr ptr) { + while (ptr && ptr->unique()) { + ptr = std::move(ptr->next_); + } + } + static void dec_ref_cnt(ChainBufferNode *ptr) { + int left = --ptr->ref_cnt_; + if (left == 0) { + clear_nonrecursive(std::move(ptr->next_)); + // TODO(refact): move memory management into allocator (?) + delete ptr; + } + } +}; + +using ChainBufferNodeWriterPtr = ChainBufferNode::WriterPtr; +using ChainBufferNodeReaderPtr = ChainBufferNode::ReaderPtr; + +class ChainBufferNodeAllocator { + public: + static ChainBufferNodeWriterPtr create(BufferSlice slice, bool sync_flag) { + auto *ptr = new ChainBufferNode(std::move(slice), sync_flag); + return ChainBufferNode::make_writer_ptr(ptr); + } + static ChainBufferNodeReaderPtr clone(const ChainBufferNodeReaderPtr &ptr) { + if (!ptr) { + return ChainBufferNodeReaderPtr(); + } + return ChainBufferNode::make_reader_ptr(ptr.get()); + } + static ChainBufferNodeReaderPtr clone(ChainBufferNodeWriterPtr &ptr) { + if (!ptr) { + return ChainBufferNodeReaderPtr(); + } + return ChainBufferNode::make_reader_ptr(ptr.get()); + } +}; + +class ChainBufferIterator { + public: + ChainBufferIterator() = default; + explicit ChainBufferIterator(ChainBufferNodeReaderPtr head) : head_(std::move(head)) { + load_head(); + } + ChainBufferIterator clone() const { + return ChainBufferIterator(ChainBufferNodeAllocator::clone(head_), reader_.clone(), need_sync_, offset_); + } + + size_t offset() const { + return offset_; + } + + void clear() { + *this = ChainBufferIterator(); + } + + Slice prepare_read() { + if (!head_) { + return Slice(); + } + while (true) { + auto res = reader_.prepare_read(); + if (!res.empty()) { + return res; + } + auto has_writer = head_->has_writer(); + if (need_sync_) { + reader_.sync_with_writer(); + res = reader_.prepare_read(); + if (!res.empty()) { + return res; + } + } + if (has_writer) { + return Slice(); + } + head_ = ChainBufferNodeAllocator::clone(head_->next_); + if (!head_) { + return Slice(); + } + load_head(); + } + } + + // returns only head + BufferSlice read_as_buffer_slice(size_t limit) { + prepare_read(); + auto res = reader_.clone(); + res.truncate(limit); + confirm_read(res.size()); + return res; + } + + const BufferSlice &head() const { + return reader_; + } + + void confirm_read(size_t size) { + offset_ += size; + reader_.confirm_read(size); + } + + void advance_till_end() { + while (true) { + auto ready = prepare_read(); + if (ready.empty()) { + break; + } + confirm_read(ready.size()); + } + } + + size_t advance(size_t offset, MutableSlice dest = MutableSlice()) { + size_t skipped = 0; + while (offset != 0) { + auto ready = prepare_read(); + if (ready.empty()) { + break; + } + + // read no more than offset + ready.truncate(offset); + offset -= ready.size(); + skipped += ready.size(); + + // copy to dest if possible + auto to_dest_size = min(ready.size(), dest.size()); + if (to_dest_size != 0) { + std::memcpy(dest.data(), ready.data(), to_dest_size); + dest.remove_prefix(to_dest_size); + } + + confirm_read(ready.size()); + } + return skipped; + } + + private: + ChainBufferNodeReaderPtr head_; + BufferSlice reader_; // copy of head_->slice_ + bool need_sync_ = false; // copy of head_->sync_flag_ + size_t offset_ = 0; // position in the union of all nodes + + ChainBufferIterator(ChainBufferNodeReaderPtr head, BufferSlice reader, bool need_sync, size_t offset) + : head_(std::move(head)), reader_(std::move(reader)), need_sync_(need_sync), offset_(offset) { + } + void load_head() { + reader_ = head_->slice_.clone(); + need_sync_ = head_->sync_flag_; + } +}; + +class ChainBufferReader { + public: + ChainBufferReader() = default; + explicit ChainBufferReader(ChainBufferNodeReaderPtr head) + : begin_(ChainBufferNodeAllocator::clone(head)), end_(std::move(head)) { + end_.advance_till_end(); + } + ChainBufferReader(ChainBufferIterator begin, ChainBufferIterator end, bool sync_flag) + : begin_(std::move(begin)), end_(std::move(end)), sync_flag_(sync_flag) { + } + ChainBufferReader(ChainBufferNodeReaderPtr head, size_t size) + : begin_(ChainBufferNodeAllocator::clone(head)), end_(std::move(head)) { + auto advanced = end_.advance(size); + CHECK(advanced == size); + } + ChainBufferReader(ChainBufferReader &&) = default; + ChainBufferReader &operator=(ChainBufferReader &&) = default; + ChainBufferReader(const ChainBufferReader &) = delete; + ChainBufferReader &operator=(const ChainBufferReader &) = delete; + ~ChainBufferReader() = default; + + ChainBufferReader clone() { + return ChainBufferReader(begin_.clone(), end_.clone(), sync_flag_); + } + + Slice prepare_read() { + auto res = begin_.prepare_read(); + res.truncate(size()); + return res; + } + + void confirm_read(size_t size) { + CHECK(size <= this->size()); + begin_.confirm_read(size); + } + + size_t advance(size_t offset, MutableSlice dest = MutableSlice()) { + CHECK(offset <= size()); + return begin_.advance(offset, dest); + } + + size_t size() const { + return end_.offset() - begin_.offset(); + } + bool empty() const { + return size() == 0; + } + + void sync_with_writer() { + if (sync_flag_) { + end_.advance_till_end(); + } + } + void advance_end(size_t size) { + end_.advance(size); + } + const ChainBufferIterator &begin() { + return begin_; + } + const ChainBufferIterator &end() { + return end_; + } + + // Return [begin_, tail.begin_) + // *this = tail + ChainBufferReader cut_head(ChainBufferIterator pos) { + auto tmp = begin_.clone(); + begin_ = pos.clone(); + return ChainBufferReader(std::move(tmp), std::move(pos), false); + } + + ChainBufferReader cut_head(size_t offset) { + CHECK(offset <= size()) << offset << " " << size(); + auto it = begin_.clone(); + it.advance(offset); + return cut_head(std::move(it)); + } + + BufferSlice move_as_buffer_slice() { + BufferSlice res; + if (begin_.head().size() >= size()) { + res = begin_.read_as_buffer_slice(size()); + } else { + auto save_size = size(); + res = BufferSlice{save_size}; + advance(save_size, res.as_slice()); + } + *this = ChainBufferReader(); + return res; + } + + BufferSlice read_as_buffer_slice(size_t limit = std::numeric_limits<size_t>::max()) { + return begin_.read_as_buffer_slice(min(limit, size())); + } + + private: + ChainBufferIterator begin_; // use it for prepare_read. Fix result with size() + ChainBufferIterator end_; // keep end as far as we can. use it for size() + bool sync_flag_ = true; // auto sync of end_ + + // 1. We have fixed size. Than end_ is useless. + // 2. No fixed size. One has to sync end_ with end_.advance_till_end() in order to calculate size. +}; + +class ChainBufferWriter { + public: + ChainBufferWriter() { + init(); + } + + // legacy + static ChainBufferWriter create_empty(size_t size = 0) { + return ChainBufferWriter(); + } + + void init(size_t size = 0) { + writer_ = BufferWriter(size); + tail_ = ChainBufferNodeAllocator::create(writer_.as_buffer_slice(), true); + head_ = ChainBufferNodeAllocator::clone(tail_); + } + + MutableSlice prepare_append(size_t hint = 0) { + CHECK(!empty()); + auto res = prepare_append_inplace(); + if (res.empty()) { + return prepare_append_alloc(hint); + } + return res; + } + MutableSlice prepare_append_inplace() { + CHECK(!empty()); + return writer_.prepare_append(); + } + MutableSlice prepare_append_alloc(size_t hint = 0) { + CHECK(!empty()); + if (hint < (1 << 10)) { + hint = 1 << 12; + } + BufferWriter new_writer(hint); + auto new_tail = ChainBufferNodeAllocator::create(new_writer.as_buffer_slice(), true); + tail_->next_ = ChainBufferNodeAllocator::clone(new_tail); + writer_ = std::move(new_writer); + tail_ = std::move(new_tail); // release tail_ + return writer_.prepare_append(); + } + void confirm_append(size_t size) { + CHECK(!empty()); + writer_.confirm_append(size); + } + + void append(Slice slice) { + while (!slice.empty()) { + auto ready = prepare_append(slice.size()); + auto shift = min(ready.size(), slice.size()); + std::memcpy(ready.data(), slice.data(), shift); + confirm_append(shift); + slice.remove_prefix(shift); + } + } + + void append(BufferSlice slice) { + auto ready = prepare_append_inplace(); + // TODO(perf): we have to store some stats in ChainBufferWriter + // for better append logic + if (slice.size() < (1 << 8) || ready.size() >= slice.size()) { + return append(slice.as_slice()); + } + + auto new_tail = ChainBufferNodeAllocator::create(std::move(slice), false); + tail_->next_ = ChainBufferNodeAllocator::clone(new_tail); + writer_ = BufferWriter(); + tail_ = std::move(new_tail); // release tail_ + } + + void append(ChainBufferReader &&reader) { + while (!reader.empty()) { + append(reader.read_as_buffer_slice()); + } + } + void append(ChainBufferReader &reader) { + while (!reader.empty()) { + append(reader.read_as_buffer_slice()); + } + } + + ChainBufferReader extract_reader() { + CHECK(head_); + return ChainBufferReader(std::move(head_)); + } + + private: + bool empty() const { + return !tail_; + } + + ChainBufferNodeReaderPtr head_; + ChainBufferNodeWriterPtr tail_; + BufferWriter writer_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/common.h b/libs/tdlib/td/tdutils/td/utils/common.h new file mode 100644 index 0000000000..d1217016e3 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/common.h @@ -0,0 +1,126 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/config.h" +#include "td/utils/port/platform.h" + +// clang-format off +#if TD_WINDOWS + #ifndef NTDDI_VERSION + #define NTDDI_VERSION 0x06020000 + #endif + #ifndef WINVER + #define WINVER 0x0602 + #endif + #ifndef _WIN32_WINNT + #define _WIN32_WINNT 0x0602 + #endif + #ifndef NOMINMAX + #define NOMINMAX + #endif + #ifndef UNICODE + #define UNICODE + #endif + #ifndef _UNICODE + #define _UNICODE + #endif + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + + #include <Winsock2.h> + #include <ws2tcpip.h> + + #include <Mswsock.h> + #include <Windows.h> + #undef ERROR +#endif +// clang-format on + +#include "td/utils/int_types.h" + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#define TD_DEBUG + +#define TD_DEFINE_STR_IMPL(x) #x +#define TD_DEFINE_STR(x) TD_DEFINE_STR_IMPL(x) +#define TD_CONCAT_IMPL(x, y) x##y +#define TD_CONCAT(x, y) TD_CONCAT_IMPL(x, y) + +// clang-format off +#if TD_WINDOWS + #define TD_DIR_SLASH '\\' +#else + #define TD_DIR_SLASH '/' +#endif +// clang-format on + +namespace td { + +inline bool likely(bool x) { +#if TD_CLANG || TD_GCC || TD_INTEL + return __builtin_expect(x, 1); +#else + return x; +#endif +} + +inline bool unlikely(bool x) { +#if TD_CLANG || TD_GCC || TD_INTEL + return __builtin_expect(x, 0); +#else + return x; +#endif +} + +// replace std::max and std::min to not have to include <algorithm> everywhere +// as a side bonus, accept parameters by value, so constexpr variables aren't required to be instantiated +template <class T> +T max(T a, T b) { + return a < b ? b : a; +} + +template <class T> +T min(T a, T b) { + return a < b ? a : b; +} + +using string = std::string; + +template <class ValueT> +using vector = std::vector<ValueT>; + +template <class ValueT> +using unique_ptr = std::unique_ptr<ValueT>; + +using std::make_unique; + +struct Unit {}; + +struct Auto { + template <class ToT> + operator ToT() const { + return ToT(); + } +}; + +template <class ToT, class FromT> +ToT &as(FromT *from) { + return *reinterpret_cast<ToT *>(from); +} + +template <class ToT, class FromT> +const ToT &as(const FromT *from) { + return *reinterpret_cast<const ToT *>(from); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/config.h b/libs/tdlib/td/tdutils/td/utils/config.h new file mode 100644 index 0000000000..ac7462480d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/config.h @@ -0,0 +1,3 @@ +#pragma once +#define TD_HAVE_OPENSSL 1 +#define TD_HAVE_ZLIB 1 diff --git a/libs/tdlib/td/tdutils/td/utils/config.h.in b/libs/tdlib/td/tdutils/td/utils/config.h.in new file mode 100644 index 0000000000..92cbd5cdc6 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/config.h.in @@ -0,0 +1,3 @@ +#pragma once +#cmakedefine01 TD_HAVE_OPENSSL +#cmakedefine01 TD_HAVE_ZLIB diff --git a/libs/tdlib/td/tdutils/td/utils/crypto.cpp b/libs/tdlib/td/tdutils/td/utils/crypto.cpp new file mode 100644 index 0000000000..3e54e673ab --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/crypto.cpp @@ -0,0 +1,541 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/crypto.h" + +#include "td/utils/BigNum.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/RwMutex.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/Random.h" + +#if TD_HAVE_OPENSSL +#include <openssl/aes.h> +#include <openssl/crypto.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/md5.h> +#include <openssl/sha.h> +#endif + +#if TD_HAVE_ZLIB +#include <zlib.h> +#endif + +#include <algorithm> +#include <cstring> +#include <utility> + +namespace td { + +static uint64 gcd(uint64 a, uint64 b) { + if (a == 0) { + return b; + } + if (b == 0) { + return a; + } + + int shift = 0; + while ((a & 1) == 0 && (b & 1) == 0) { + a >>= 1; + b >>= 1; + shift++; + } + + while (true) { + while ((a & 1) == 0) { + a >>= 1; + } + while ((b & 1) == 0) { + b >>= 1; + } + if (a > b) { + a -= b; + } else if (b > a) { + b -= a; + } else { + return a << shift; + } + } +} + +uint64 pq_factorize(uint64 pq) { + if (pq < 2 || pq > (static_cast<uint64>(1) << 63)) { + return 1; + } + uint64 g = 0; + for (int i = 0, it = 0; i < 3 || it < 1000; i++) { + uint64 q = Random::fast(17, 32) % (pq - 1); + uint64 x = Random::fast_uint64() % (pq - 1) + 1; + uint64 y = x; + int lim = 1 << (min(5, i) + 18); + for (int j = 1; j < lim; j++) { + it++; + uint64 a = x; + uint64 b = x; + uint64 c = q; + + // c += a * b + while (b) { + if (b & 1) { + c += a; + if (c >= pq) { + c -= pq; + } + } + a += a; + if (a >= pq) { + a -= pq; + } + b >>= 1; + } + + x = c; + uint64 z = x < y ? pq + x - y : x - y; + g = gcd(z, pq); + if (g != 1) { + break; + } + + if (!(j & (j - 1))) { + y = x; + } + } + if (g > 1 && g < pq) { + break; + } + } + if (g != 0) { + uint64 other = pq / g; + if (other < g) { + g = other; + } + } + return g; +} + +#if TD_HAVE_OPENSSL +void init_crypto() { + static bool is_inited = [] { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + return OPENSSL_init_crypto(0, nullptr) != 0; +#else + OpenSSL_add_all_algorithms(); + return true; +#endif + }(); + CHECK(is_inited); +} + +template <class FromT> +static string as_big_endian_string(const FromT &from) { + size_t size = sizeof(from); + string res(size, '\0'); + + auto ptr = reinterpret_cast<const unsigned char *>(&from); + std::memcpy(&res[0], ptr, size); + + size_t i = size; + while (i && res[i - 1] == 0) { + i--; + } + + res.resize(i); + std::reverse(res.begin(), res.end()); + return res; +} + +static int pq_factorize_big(Slice pq_str, string *p_str, string *q_str) { + // TODO: qsieve? + // do not work for pq == 1 + BigNumContext context; + BigNum a; + BigNum b; + BigNum p; + BigNum q; + BigNum one; + one.set_value(1); + + BigNum pq = BigNum::from_binary(pq_str); + + bool found = false; + for (int i = 0, it = 0; !found && (i < 3 || it < 1000); i++) { + int32 t = Random::fast(17, 32); + a.set_value(Random::fast_uint32()); + b = a; + + int32 lim = 1 << (i + 23); + for (int j = 1; j < lim; j++) { + it++; + BigNum::mod_mul(a, a, a, pq, context); + a += t; + if (BigNum::compare(a, pq) >= 0) { + BigNum tmp; + BigNum::sub(tmp, a, pq); + a = std::move(tmp); + } + if (BigNum::compare(a, b) > 0) { + BigNum::sub(q, a, b); + } else { + BigNum::sub(q, b, a); + } + BigNum::gcd(p, q, pq, context); + if (BigNum::compare(p, one) != 0) { + found = true; + break; + } + if ((j & (j - 1)) == 0) { + b = a; + } + } + } + + if (found) { + BigNum::div(&q, nullptr, pq, p, context); + if (BigNum::compare(p, q) > 0) { + std::swap(p, q); + } + + *p_str = p.to_binary(); + *q_str = q.to_binary(); + + return 0; + } + + return -1; +} + +int pq_factorize(Slice pq_str, string *p_str, string *q_str) { + size_t size = pq_str.size(); + if (static_cast<int>(size) > 8 || (static_cast<int>(size) == 8 && (pq_str.begin()[0] & 128) != 0)) { + return pq_factorize_big(pq_str, p_str, q_str); + } + + auto ptr = pq_str.ubegin(); + uint64 pq = 0; + for (int i = 0; i < static_cast<int>(size); i++) { + pq = (pq << 8) | ptr[i]; + } + + uint64 p = pq_factorize(pq); + if (p == 0 || pq % p != 0) { + return -1; + } + *p_str = as_big_endian_string(p); + *q_str = as_big_endian_string(pq / p); + + // std::string p2, q2; + // pq_factorize_big(pq_str, &p2, &q2); + // CHECK(*p_str == p2); + // CHECK(*q_str == q2); + return 0; +} + +/*** AES ***/ +static void aes_ige_xcrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { + AES_KEY key; + int err; + if (encrypt_flag) { + err = AES_set_encrypt_key(aes_key.raw, 256, &key); + } else { + err = AES_set_decrypt_key(aes_key.raw, 256, &key); + } + LOG_IF(FATAL, err != 0); + CHECK(from.size() <= to.size()); + AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag); +} + +void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) { + aes_ige_xcrypt(aes_key, aes_iv, from, to, true); +} + +void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to) { + aes_ige_xcrypt(aes_key, aes_iv, from, to, false); +} + +static void aes_cbc_xcrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { + AES_KEY key; + int err; + if (encrypt_flag) { + err = AES_set_encrypt_key(aes_key.raw, 256, &key); + } else { + err = AES_set_decrypt_key(aes_key.raw, 256, &key); + } + LOG_IF(FATAL, err != 0); + CHECK(from.size() <= to.size()); + AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv->raw, encrypt_flag); +} + +void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) { + aes_cbc_xcrypt(aes_key, aes_iv, from, to, true); +} + +void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to) { + aes_cbc_xcrypt(aes_key, aes_iv, from, to, false); +} + +class AesCtrState::Impl { + public: + Impl(const UInt256 &key, const UInt128 &iv) { + if (AES_set_encrypt_key(key.raw, 256, &aes_key) < 0) { + LOG(FATAL) << "Failed to set encrypt key"; + } + MutableSlice(counter, AES_BLOCK_SIZE).copy_from({iv.raw, AES_BLOCK_SIZE}); + current_pos = 0; + } + + void encrypt(Slice from, MutableSlice to) { + CHECK(to.size() >= from.size()); + for (size_t i = 0; i < from.size(); i++) { + if (current_pos == 0) { + AES_encrypt(counter, encrypted_counter, &aes_key); + for (int j = 15; j >= 0; j--) { + if (++counter[j] != 0) { + break; + } + } + } + to[i] = static_cast<char>(from[i] ^ encrypted_counter[current_pos]); + current_pos = (current_pos + 1) & 15; + } + } + + private: + AES_KEY aes_key; + uint8 counter[AES_BLOCK_SIZE]; + uint8 encrypted_counter[AES_BLOCK_SIZE]; + uint8 current_pos; +}; + +AesCtrState::AesCtrState() = default; +AesCtrState::AesCtrState(AesCtrState &&from) = default; +AesCtrState &AesCtrState::operator=(AesCtrState &&from) = default; +AesCtrState::~AesCtrState() = default; + +void AesCtrState::init(const UInt256 &key, const UInt128 &iv) { + ctx_ = std::make_unique<AesCtrState::Impl>(key, iv); +} + +void AesCtrState::encrypt(Slice from, MutableSlice to) { + ctx_->encrypt(from, to); +} + +void AesCtrState::decrypt(Slice from, MutableSlice to) { + encrypt(from, to); // it is the same as decrypt +} + +void sha1(Slice data, unsigned char output[20]) { + auto result = SHA1(data.ubegin(), data.size(), output); + CHECK(result == output); +} + +void sha256(Slice data, MutableSlice output) { + CHECK(output.size() >= 32); + auto result = SHA256(data.ubegin(), data.size(), output.ubegin()); + CHECK(result == output.ubegin()); +} + +struct Sha256StateImpl { + SHA256_CTX ctx; +}; + +Sha256State::Sha256State() = default; +Sha256State::Sha256State(Sha256State &&from) = default; +Sha256State &Sha256State::operator=(Sha256State &&from) = default; +Sha256State::~Sha256State() = default; + +void sha256_init(Sha256State *state) { + state->impl = std::make_unique<Sha256StateImpl>(); + int err = SHA256_Init(&state->impl->ctx); + LOG_IF(FATAL, err != 1); +} + +void sha256_update(Slice data, Sha256State *state) { + CHECK(state->impl); + int err = SHA256_Update(&state->impl->ctx, data.ubegin(), data.size()); + LOG_IF(FATAL, err != 1); +} + +void sha256_final(Sha256State *state, MutableSlice output) { + CHECK(output.size() >= 32); + CHECK(state->impl); + int err = SHA256_Final(output.ubegin(), &state->impl->ctx); + LOG_IF(FATAL, err != 1); + state->impl.reset(); +} + +/*** md5 ***/ +void md5(Slice input, MutableSlice output) { + CHECK(output.size() >= MD5_DIGEST_LENGTH); + auto result = MD5(input.ubegin(), input.size(), output.ubegin()); + CHECK(result == output.ubegin()); +} + +void pbkdf2_sha256(Slice password, Slice salt, int iteration_count, MutableSlice dest) { + CHECK(dest.size() == 256 / 8) << dest.size(); + CHECK(iteration_count > 0); + auto evp_md = EVP_sha256(); + CHECK(evp_md != nullptr); +#if OPENSSL_VERSION_NUMBER < 0x10000000L + HMAC_CTX ctx; + HMAC_CTX_init(&ctx); + unsigned char counter[4] = {0, 0, 0, 1}; + int password_len = narrow_cast<int>(password.size()); + HMAC_Init_ex(&ctx, password.data(), password_len, evp_md, nullptr); + HMAC_Update(&ctx, salt.ubegin(), narrow_cast<int>(salt.size())); + HMAC_Update(&ctx, counter, 4); + HMAC_Final(&ctx, dest.ubegin(), nullptr); + HMAC_CTX_cleanup(&ctx); + + if (iteration_count > 1) { + unsigned char buf[32]; + std::copy(dest.ubegin(), dest.uend(), buf); + for (int iter = 1; iter < iteration_count; iter++) { + if (HMAC(evp_md, password.data(), password_len, buf, 32, buf, nullptr) == nullptr) { + LOG(FATAL) << "Failed to HMAC"; + } + for (int i = 0; i < 32; i++) { + dest[i] ^= buf[i]; + } + } + } +#else + int err = PKCS5_PBKDF2_HMAC(password.data(), narrow_cast<int>(password.size()), salt.ubegin(), + narrow_cast<int>(salt.size()), iteration_count, evp_md, narrow_cast<int>(dest.size()), + dest.ubegin()); + LOG_IF(FATAL, err != 1); +#endif +} + +void hmac_sha256(Slice key, Slice message, MutableSlice dest) { + CHECK(dest.size() == 256 / 8); + unsigned int len = 0; + auto result = HMAC(EVP_sha256(), key.ubegin(), narrow_cast<int>(key.size()), message.ubegin(), + narrow_cast<int>(message.size()), dest.ubegin(), &len); + CHECK(result == dest.ubegin()); + CHECK(len == dest.size()); +} + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +namespace { +std::vector<RwMutex> &openssl_mutexes() { + static std::vector<RwMutex> mutexes(CRYPTO_num_locks()); + return mutexes; +} + +#if OPENSSL_VERSION_NUMBER >= 0x10000000L +void openssl_threadid_callback(CRYPTO_THREADID *thread_id) { + static TD_THREAD_LOCAL int id; + CRYPTO_THREADID_set_pointer(thread_id, &id); +} +#endif + +void openssl_locking_function(int mode, int n, const char *file, int line) { + auto &mutexes = openssl_mutexes(); + if (mode & CRYPTO_LOCK) { + if (mode & CRYPTO_READ) { + mutexes[n].lock_read_unsafe(); + } else { + mutexes[n].lock_write_unsafe(); + } + } else { + if (mode & CRYPTO_READ) { + mutexes[n].unlock_read_unsafe(); + } else { + mutexes[n].unlock_write_unsafe(); + } + } +} +} // namespace +#endif + +void init_openssl_threads() { +#if OPENSSL_VERSION_NUMBER < 0x10100000L + if (CRYPTO_get_locking_callback() == nullptr) { +#if OPENSSL_VERSION_NUMBER >= 0x10000000L + CRYPTO_THREADID_set_callback(openssl_threadid_callback); +#endif + CRYPTO_set_locking_callback(openssl_locking_function); + } +#endif +} +#endif + +#if TD_HAVE_ZLIB +uint32 crc32(Slice data) { + return static_cast<uint32>(::crc32(0, data.ubegin(), static_cast<uint32>(data.size()))); +} +#endif + +static const uint64 crc64_table[256] = { + 0x0000000000000000, 0xb32e4cbe03a75f6f, 0xf4843657a840a05b, 0x47aa7ae9abe7ff34, 0x7bd0c384ff8f5e33, + 0xc8fe8f3afc28015c, 0x8f54f5d357cffe68, 0x3c7ab96d5468a107, 0xf7a18709ff1ebc66, 0x448fcbb7fcb9e309, + 0x0325b15e575e1c3d, 0xb00bfde054f94352, 0x8c71448d0091e255, 0x3f5f08330336bd3a, 0x78f572daa8d1420e, + 0xcbdb3e64ab761d61, 0x7d9ba13851336649, 0xceb5ed8652943926, 0x891f976ff973c612, 0x3a31dbd1fad4997d, + 0x064b62bcaebc387a, 0xb5652e02ad1b6715, 0xf2cf54eb06fc9821, 0x41e11855055bc74e, 0x8a3a2631ae2dda2f, + 0x39146a8fad8a8540, 0x7ebe1066066d7a74, 0xcd905cd805ca251b, 0xf1eae5b551a2841c, 0x42c4a90b5205db73, + 0x056ed3e2f9e22447, 0xb6409f5cfa457b28, 0xfb374270a266cc92, 0x48190ecea1c193fd, 0x0fb374270a266cc9, + 0xbc9d3899098133a6, 0x80e781f45de992a1, 0x33c9cd4a5e4ecdce, 0x7463b7a3f5a932fa, 0xc74dfb1df60e6d95, + 0x0c96c5795d7870f4, 0xbfb889c75edf2f9b, 0xf812f32ef538d0af, 0x4b3cbf90f69f8fc0, 0x774606fda2f72ec7, + 0xc4684a43a15071a8, 0x83c230aa0ab78e9c, 0x30ec7c140910d1f3, 0x86ace348f355aadb, 0x3582aff6f0f2f5b4, + 0x7228d51f5b150a80, 0xc10699a158b255ef, 0xfd7c20cc0cdaf4e8, 0x4e526c720f7dab87, 0x09f8169ba49a54b3, + 0xbad65a25a73d0bdc, 0x710d64410c4b16bd, 0xc22328ff0fec49d2, 0x85895216a40bb6e6, 0x36a71ea8a7ace989, + 0x0adda7c5f3c4488e, 0xb9f3eb7bf06317e1, 0xfe5991925b84e8d5, 0x4d77dd2c5823b7ba, 0x64b62bcaebc387a1, + 0xd7986774e864d8ce, 0x90321d9d438327fa, 0x231c512340247895, 0x1f66e84e144cd992, 0xac48a4f017eb86fd, + 0xebe2de19bc0c79c9, 0x58cc92a7bfab26a6, 0x9317acc314dd3bc7, 0x2039e07d177a64a8, 0x67939a94bc9d9b9c, + 0xd4bdd62abf3ac4f3, 0xe8c76f47eb5265f4, 0x5be923f9e8f53a9b, 0x1c4359104312c5af, 0xaf6d15ae40b59ac0, + 0x192d8af2baf0e1e8, 0xaa03c64cb957be87, 0xeda9bca512b041b3, 0x5e87f01b11171edc, 0x62fd4976457fbfdb, + 0xd1d305c846d8e0b4, 0x96797f21ed3f1f80, 0x2557339fee9840ef, 0xee8c0dfb45ee5d8e, 0x5da24145464902e1, + 0x1a083bacedaefdd5, 0xa9267712ee09a2ba, 0x955cce7fba6103bd, 0x267282c1b9c65cd2, 0x61d8f8281221a3e6, + 0xd2f6b4961186fc89, 0x9f8169ba49a54b33, 0x2caf25044a02145c, 0x6b055fede1e5eb68, 0xd82b1353e242b407, + 0xe451aa3eb62a1500, 0x577fe680b58d4a6f, 0x10d59c691e6ab55b, 0xa3fbd0d71dcdea34, 0x6820eeb3b6bbf755, + 0xdb0ea20db51ca83a, 0x9ca4d8e41efb570e, 0x2f8a945a1d5c0861, 0x13f02d374934a966, 0xa0de61894a93f609, + 0xe7741b60e174093d, 0x545a57dee2d35652, 0xe21ac88218962d7a, 0x5134843c1b317215, 0x169efed5b0d68d21, + 0xa5b0b26bb371d24e, 0x99ca0b06e7197349, 0x2ae447b8e4be2c26, 0x6d4e3d514f59d312, 0xde6071ef4cfe8c7d, + 0x15bb4f8be788911c, 0xa6950335e42fce73, 0xe13f79dc4fc83147, 0x521135624c6f6e28, 0x6e6b8c0f1807cf2f, + 0xdd45c0b11ba09040, 0x9aefba58b0476f74, 0x29c1f6e6b3e0301b, 0xc96c5795d7870f42, 0x7a421b2bd420502d, + 0x3de861c27fc7af19, 0x8ec62d7c7c60f076, 0xb2bc941128085171, 0x0192d8af2baf0e1e, 0x4638a2468048f12a, + 0xf516eef883efae45, 0x3ecdd09c2899b324, 0x8de39c222b3eec4b, 0xca49e6cb80d9137f, 0x7967aa75837e4c10, + 0x451d1318d716ed17, 0xf6335fa6d4b1b278, 0xb199254f7f564d4c, 0x02b769f17cf11223, 0xb4f7f6ad86b4690b, + 0x07d9ba1385133664, 0x4073c0fa2ef4c950, 0xf35d8c442d53963f, 0xcf273529793b3738, 0x7c0979977a9c6857, + 0x3ba3037ed17b9763, 0x888d4fc0d2dcc80c, 0x435671a479aad56d, 0xf0783d1a7a0d8a02, 0xb7d247f3d1ea7536, + 0x04fc0b4dd24d2a59, 0x3886b22086258b5e, 0x8ba8fe9e8582d431, 0xcc0284772e652b05, 0x7f2cc8c92dc2746a, + 0x325b15e575e1c3d0, 0x8175595b76469cbf, 0xc6df23b2dda1638b, 0x75f16f0cde063ce4, 0x498bd6618a6e9de3, + 0xfaa59adf89c9c28c, 0xbd0fe036222e3db8, 0x0e21ac88218962d7, 0xc5fa92ec8aff7fb6, 0x76d4de52895820d9, + 0x317ea4bb22bfdfed, 0x8250e80521188082, 0xbe2a516875702185, 0x0d041dd676d77eea, 0x4aae673fdd3081de, + 0xf9802b81de97deb1, 0x4fc0b4dd24d2a599, 0xfceef8632775faf6, 0xbb44828a8c9205c2, 0x086ace348f355aad, + 0x34107759db5dfbaa, 0x873e3be7d8faa4c5, 0xc094410e731d5bf1, 0x73ba0db070ba049e, 0xb86133d4dbcc19ff, + 0x0b4f7f6ad86b4690, 0x4ce50583738cb9a4, 0xffcb493d702be6cb, 0xc3b1f050244347cc, 0x709fbcee27e418a3, + 0x3735c6078c03e797, 0x841b8ab98fa4b8f8, 0xadda7c5f3c4488e3, 0x1ef430e13fe3d78c, 0x595e4a08940428b8, + 0xea7006b697a377d7, 0xd60abfdbc3cbd6d0, 0x6524f365c06c89bf, 0x228e898c6b8b768b, 0x91a0c532682c29e4, + 0x5a7bfb56c35a3485, 0xe955b7e8c0fd6bea, 0xaeffcd016b1a94de, 0x1dd181bf68bdcbb1, 0x21ab38d23cd56ab6, + 0x9285746c3f7235d9, 0xd52f0e859495caed, 0x6601423b97329582, 0xd041dd676d77eeaa, 0x636f91d96ed0b1c5, + 0x24c5eb30c5374ef1, 0x97eba78ec690119e, 0xab911ee392f8b099, 0x18bf525d915feff6, 0x5f1528b43ab810c2, + 0xec3b640a391f4fad, 0x27e05a6e926952cc, 0x94ce16d091ce0da3, 0xd3646c393a29f297, 0x604a2087398eadf8, + 0x5c3099ea6de60cff, 0xef1ed5546e415390, 0xa8b4afbdc5a6aca4, 0x1b9ae303c601f3cb, 0x56ed3e2f9e224471, + 0xe5c372919d851b1e, 0xa26908783662e42a, 0x114744c635c5bb45, 0x2d3dfdab61ad1a42, 0x9e13b115620a452d, + 0xd9b9cbfcc9edba19, 0x6a978742ca4ae576, 0xa14cb926613cf817, 0x1262f598629ba778, 0x55c88f71c97c584c, + 0xe6e6c3cfcadb0723, 0xda9c7aa29eb3a624, 0x69b2361c9d14f94b, 0x2e184cf536f3067f, 0x9d36004b35545910, + 0x2b769f17cf112238, 0x9858d3a9ccb67d57, 0xdff2a94067518263, 0x6cdce5fe64f6dd0c, 0x50a65c93309e7c0b, + 0xe388102d33392364, 0xa4226ac498dedc50, 0x170c267a9b79833f, 0xdcd7181e300f9e5e, 0x6ff954a033a8c131, + 0x28532e49984f3e05, 0x9b7d62f79be8616a, 0xa707db9acf80c06d, 0x14299724cc279f02, 0x5383edcd67c06036, + 0xe0ada17364673f59}; + +static uint64 crc64_partial(Slice data, uint64 crc) { + const char *p = data.begin(); + for (auto len = data.size(); len > 0; len--) { + crc = crc64_table[(crc ^ *p++) & 0xff] ^ (crc >> 8); + } + return crc; +} + +uint64 crc64(Slice data) { + return crc64_partial(data, static_cast<uint64>(-1)) ^ static_cast<uint64>(-1); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/crypto.h b/libs/tdlib/td/tdutils/td/utils/crypto.h new file mode 100644 index 0000000000..23ac694bfb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/crypto.h @@ -0,0 +1,79 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +uint64 pq_factorize(uint64 pq); + +#if TD_HAVE_OPENSSL +void init_crypto(); + +int pq_factorize(Slice pq_str, string *p_str, string *q_str); + +void aes_ige_encrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to); +void aes_ige_decrypt(const UInt256 &aes_key, UInt256 *aes_iv, Slice from, MutableSlice to); + +void aes_cbc_encrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to); +void aes_cbc_decrypt(const UInt256 &aes_key, UInt128 *aes_iv, Slice from, MutableSlice to); + +class AesCtrState { + public: + AesCtrState(); + AesCtrState(const AesCtrState &from) = delete; + AesCtrState &operator=(const AesCtrState &from) = delete; + AesCtrState(AesCtrState &&from); + AesCtrState &operator=(AesCtrState &&from); + ~AesCtrState(); + + void init(const UInt256 &key, const UInt128 &iv); + + void encrypt(Slice from, MutableSlice to); + + void decrypt(Slice from, MutableSlice to); + + private: + class Impl; + std::unique_ptr<Impl> ctx_; +}; + +void sha1(Slice data, unsigned char output[20]); + +void sha256(Slice data, MutableSlice output); + +struct Sha256StateImpl; + +struct Sha256State { + Sha256State(); + Sha256State(Sha256State &&from); + Sha256State &operator=(Sha256State &&from); + ~Sha256State(); + std::unique_ptr<Sha256StateImpl> impl; +}; + +void sha256_init(Sha256State *state); +void sha256_update(Slice data, Sha256State *state); +void sha256_final(Sha256State *state, MutableSlice output); + +void md5(Slice input, MutableSlice output); + +void pbkdf2_sha256(Slice password, Slice salt, int iteration_count, MutableSlice dest); +void hmac_sha256(Slice key, Slice message, MutableSlice dest); + +void init_openssl_threads(); +#endif + +#if TD_HAVE_ZLIB +uint32 crc32(Slice data); +#endif + +uint64 crc64(Slice data); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/filesystem.cpp b/libs/tdlib/td/tdutils/td/utils/filesystem.cpp new file mode 100644 index 0000000000..b22418151c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/filesystem.cpp @@ -0,0 +1,123 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/filesystem.h" + +#include "td/utils/buffer.h" +#include "td/utils/logging.h" +#include "td/utils/PathView.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/unicode.h" +#include "td/utils/utf8.h" + +namespace td { + +Result<BufferSlice> read_file(CSlice path, int64 size) { + TRY_RESULT(from_file, FileFd::open(path, FileFd::Read)); + if (size == -1) { + size = from_file.get_size(); + } + BufferWriter content{static_cast<size_t>(size), 0, 0}; + TRY_RESULT(got_size, from_file.read(content.as_slice())); + if (got_size != static_cast<size_t>(size)) { + return Status::Error("Failed to read file"); + } + from_file.close(); + return content.as_buffer_slice(); +} + +// Very straightforward function. Don't expect much of it. +Status copy_file(CSlice from, CSlice to, int64 size) { + TRY_RESULT(content, read_file(from, size)); + return write_file(to, content.as_slice()); +} + +Status write_file(CSlice to, Slice data) { + auto size = data.size(); + TRY_RESULT(to_file, FileFd::open(to, FileFd::Truncate | FileFd::Create | FileFd::Write)); + TRY_RESULT(written, to_file.write(data)); + if (written != static_cast<size_t>(size)) { + return Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size); + } + to_file.close(); + return Status::OK(); +} + +static std::string clean_filename_part(Slice name, int max_length) { + auto is_ok = [](uint32 code) { + if (code < 32) { + return false; + } + if (code < 127) { + switch (code) { + case '<': + case '>': + case ':': + case '"': + case '/': + case '\\': + case '|': + case '?': + case '*': + case '&': + case '`': + case '\'': + return false; + default: + return true; + } + } + auto category = get_unicode_simple_category(code); + + return category == UnicodeSimpleCategory::Letter || category == UnicodeSimpleCategory::DecimalNumber || + category == UnicodeSimpleCategory::Number; + }; + + std::string new_name; + int size = 0; + for (auto *it = name.ubegin(); it != name.uend() && size < max_length;) { + uint32 code; + it = next_utf8_unsafe(it, &code); + if (!is_ok(code)) { + code = ' '; + } + if (new_name.empty() && (code == ' ' || code == '.')) { + continue; + } + append_utf8_character(new_name, code); + size++; + } + + while (!new_name.empty() && (new_name.back() == ' ' || new_name.back() == '.')) { + new_name.pop_back(); + } + return new_name; +} + +std::string clean_filename(CSlice name) { + if (!check_utf8(name)) { + return {}; + } + + PathView path_view(name); + auto filename = clean_filename_part(path_view.file_stem(), 60); + auto extension = clean_filename_part(path_view.extension(), 20); + if (!extension.empty()) { + if (filename.empty()) { + filename = std::move(extension); + } else { + filename.reserve(filename.size() + 1 + extension.size()); + filename += '.'; + filename += extension; + } + } + + return filename; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/filesystem.h b/libs/tdlib/td/tdutils/td/utils/filesystem.h new file mode 100644 index 0000000000..4bb1b17191 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/filesystem.h @@ -0,0 +1,22 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +Result<BufferSlice> read_file(CSlice path, int64 size = -1); + +Status copy_file(CSlice from, CSlice to, int64 size = -1); + +Status write_file(CSlice to, Slice data); + +std::string clean_filename(CSlice name); +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/find_boundary.cpp b/libs/tdlib/td/tdutils/td/utils/find_boundary.cpp new file mode 100644 index 0000000000..44fc264ab5 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/find_boundary.cpp @@ -0,0 +1,53 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/find_boundary.h" + +#include "td/utils/logging.h" + +#include <cstring> + +namespace td { + +bool find_boundary(ChainBufferReader range, Slice boundary, size_t &already_read) { + range.advance(already_read); + + const int MAX_BOUNDARY_LENGTH = 70; + CHECK(boundary.size() <= MAX_BOUNDARY_LENGTH + 4); + while (!range.empty()) { + Slice ready = range.prepare_read(); + if (ready[0] == boundary[0]) { + if (range.size() < boundary.size()) { + return false; + } + auto save_range = range.clone(); + char x[MAX_BOUNDARY_LENGTH + 4]; + range.advance(boundary.size(), {x, sizeof(x)}); + if (std::memcmp(x, boundary.data(), boundary.size()) == 0) { + return true; + } + + // not a boundary, restoring previous state and skip one symbol + range = std::move(save_range); + range.advance(1); + already_read++; + } else { + const char *ptr = static_cast<const char *>(std::memchr(ready.data(), boundary[0], ready.size())); + size_t shift; + if (ptr == nullptr) { + shift = ready.size(); + } else { + shift = ptr - ready.data(); + } + already_read += shift; + range.advance(shift); + } + } + + return false; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/find_boundary.h b/libs/tdlib/td/tdutils/td/utils/find_boundary.h new file mode 100644 index 0000000000..5b424cf23c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/find_boundary.h @@ -0,0 +1,17 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/Slice.h" + +namespace td { + +bool find_boundary(ChainBufferReader range, Slice boundary, size_t &already_read); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/format.h b/libs/tdlib/td/tdutils/td/utils/format.h new file mode 100644 index 0000000000..745ad0d8a5 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/format.h @@ -0,0 +1,312 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/invoke.h" // for tuple_for_each +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" + +#include <tuple> +#include <utility> + +namespace td { +namespace format { +/*** HexDump ***/ +template <std::size_t size, bool reversed = true> +struct HexDumpSize { + const unsigned char *data; +}; + +inline char hex_digit(int x) { + return "0123456789abcdef"[x]; +} + +template <std::size_t size, bool reversed> +StringBuilder &operator<<(StringBuilder &builder, const HexDumpSize<size, reversed> &dump) { + const unsigned char *ptr = dump.data; + // TODO: append unsafe + for (std::size_t i = 0; i < size; i++) { + int xy = ptr[reversed ? size - 1 - i : i]; + int x = xy >> 4; + int y = xy & 15; + builder << hex_digit(x) << hex_digit(y); + } + return builder; +} + +template <std::size_t align> +struct HexDumpSlice { + Slice slice; +}; + +template <std::size_t align> +StringBuilder &operator<<(StringBuilder &builder, const HexDumpSlice<align> &dump) { + std::size_t size = dump.slice.size(); + const unsigned char *ptr = dump.slice.ubegin(); + + builder << '\n'; + + const std::size_t part = size % align; + if (part) { + builder << HexDumpSlice<1>{Slice(ptr, part)} << '\n'; + } + size -= part; + ptr += part; + + for (std::size_t i = 0; i < size; i += align) { + builder << HexDumpSize<align>{ptr}; + ptr += align; + + if (((i / align) & 15) == 15 || i + align >= size) { + builder << '\n'; + } else { + builder << ' '; + } + } + + return builder; +} + +inline StringBuilder &operator<<(StringBuilder &builder, const HexDumpSlice<0> &dump) { + auto size = dump.slice.size(); + const uint8 *ptr = dump.slice.ubegin(); + for (size_t i = 0; i < size; i++) { + builder << HexDumpSize<1>{ptr + i}; + } + return builder; +} + +template <std::size_t align> +HexDumpSlice<align> as_hex_dump(Slice slice) { + return HexDumpSlice<align>{slice}; +} + +template <std::size_t align> +HexDumpSlice<align> as_hex_dump(MutableSlice slice) { + return HexDumpSlice<align>{slice}; +} + +template <std::size_t align, class T> +HexDumpSlice<align> as_hex_dump(const T &value) { + return HexDumpSlice<align>{Slice(&value, sizeof(value))}; +} +template <class T> +HexDumpSize<sizeof(T), true> as_hex_dump(const T &value) { + return HexDumpSize<sizeof(T), true>{reinterpret_cast<const unsigned char *>(&value)}; +} + +/*** Hex ***/ +template <class T> +struct Hex { + const T &value; +}; + +template <class T> +Hex<T> as_hex(const T &value) { + return Hex<T>{value}; +} + +template <class T> +StringBuilder &operator<<(StringBuilder &builder, const Hex<T> &hex) { + builder << "0x" << as_hex_dump(hex.value); + return builder; +} + +/*** Binary ***/ +template <class T> +struct Binary { + const T &value; +}; + +template <class T> +Binary<T> as_binary(const T &value) { + return Binary<T>{value}; +} + +template <class T> +StringBuilder &operator<<(StringBuilder &builder, const Binary<T> &hex) { + for (size_t i = 0; i < sizeof(T) * 8; i++) { + builder << ((hex.value >> i) & 1 ? '1' : '0'); + } + return builder; +} + +/*** Escaped ***/ +struct Escaped { + Slice str; +}; + +inline StringBuilder &operator<<(StringBuilder &builder, const Escaped &escaped) { + Slice str = escaped.str; + for (unsigned char c : str) { + if (c > 31 && c < 127 && c != '"' && c != '\\') { + builder << static_cast<char>(c); + } else { + const char *oct = "01234567"; + builder << "\\0" << oct[c >> 6] << oct[(c >> 3) & 7] << oct[c & 7]; + } + } + return builder; +} + +inline Escaped escaped(Slice slice) { + return Escaped{slice}; +} + +/*** Time to string ***/ +struct Time { + double seconds_; +}; + +inline StringBuilder &operator<<(StringBuilder &logger, Time t) { + struct NamedValue { + const char *name; + double value; + }; + + static constexpr NamedValue durations[] = {{"ns", 1e-9}, {"us", 1e-6}, {"ms", 1e-3}, {"s", 1}}; + static constexpr size_t durations_n = sizeof(durations) / sizeof(NamedValue); + + size_t i = 0; + while (i + 1 < durations_n && t.seconds_ > 10 * durations[i + 1].value) { + i++; + } + logger << StringBuilder::FixedDouble(t.seconds_ / durations[i].value, 1) << durations[i].name; + return logger; +} + +inline Time as_time(double seconds) { + return Time{seconds}; +} + +/*** Size to string ***/ +struct Size { + uint64 size_; +}; + +inline StringBuilder &operator<<(StringBuilder &logger, Size t) { + struct NamedValue { + const char *name; + uint64 value; + }; + + static constexpr NamedValue sizes[] = {{"B", 1}, {"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}}; + static constexpr size_t sizes_n = sizeof(sizes) / sizeof(NamedValue); + + size_t i = 0; + while (i + 1 < sizes_n && t.size_ > 10 * sizes[i + 1].value) { + i++; + } + logger << t.size_ / sizes[i].value << sizes[i].name; + return logger; +} + +inline Size as_size(uint64 size) { + return Size{size}; +} + +/*** Array to string ***/ +template <class ArrayT> +struct Array { + const ArrayT &ref; +}; + +template <class ArrayT> +StringBuilder &operator<<(StringBuilder &stream, const Array<ArrayT> &array) { + bool first = true; + stream << Slice("{"); + for (auto &x : array.ref) { + if (!first) { + stream << Slice(", "); + } + stream << x; + first = false; + } + return stream << Slice("}"); +} + +template <class ArrayT> +Array<ArrayT> as_array(const ArrayT &array) { + return Array<ArrayT>{array}; +} + +/*** Tagged ***/ +template <class ValueT> +struct Tagged { + Slice tag; + const ValueT &ref; +}; + +template <class ValueT> +StringBuilder &operator<<(StringBuilder &stream, const Tagged<ValueT> &tagged) { + return stream << "[" << tagged.tag << ":" << tagged.ref << "]"; +} + +template <class ValueT> +Tagged<ValueT> tag(Slice tag, const ValueT &ref) { + return Tagged<ValueT>{tag, ref}; +} + +/*** Cond ***/ +inline StringBuilder &operator<<(StringBuilder &sb, Unit) { + return sb; +} + +template <class TrueT, class FalseT> +struct Cond { + bool flag; + const TrueT &on_true; + const FalseT &on_false; +}; + +template <class TrueT, class FalseT> +StringBuilder &operator<<(StringBuilder &sb, const Cond<TrueT, FalseT> &cond) { + if (cond.flag) { + return sb << cond.on_true; + } else { + return sb << cond.on_false; + } +} + +template <class TrueT, class FalseT = Unit> +Cond<TrueT, FalseT> cond(bool flag, const TrueT &on_true, const FalseT &on_false = FalseT()) { + return Cond<TrueT, FalseT>{flag, on_true, on_false}; +} + +/*** Concat ***/ +template <class T> +struct Concat { + T args; +}; + +template <class T> +StringBuilder &operator<<(StringBuilder &sb, const Concat<T> &concat) { + tuple_for_each(concat.args, [&sb](auto &x) { sb << x; }); + return sb; +} + +template <class... ArgsT> +auto concat(const ArgsT &... args) { + return Concat<decltype(std::tie(args...))>{std::tie(args...)}; +} + +} // namespace format + +using format::tag; + +template <class A, class B> +StringBuilder &operator<<(StringBuilder &sb, const std::pair<A, B> &p) { + return sb << "[" << p.first << ";" << p.second << "]"; +} + +template <class T> +StringBuilder &operator<<(StringBuilder &stream, const vector<T> &vec) { + return stream << format::as_array(vec); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/int_types.h b/libs/tdlib/td/tdutils/td/utils/int_types.h new file mode 100644 index 0000000000..08ff1099c2 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/int_types.h @@ -0,0 +1,65 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/platform.h" + +#include <cstddef> +#include <cstdint> +#include <cstring> + +namespace td { + +#if !TD_WINDOWS +using size_t = std::size_t; +#endif + +using int8 = std::int8_t; +using int16 = std::int16_t; +using uint16 = std::uint16_t; +using int32 = std::int32_t; +using uint32 = std::uint32_t; +using int64 = std::int64_t; +using uint64 = std::uint64_t; + +static_assert(sizeof(std::uint8_t) == sizeof(unsigned char), "Unsigned char expected to be 8-bit"); +using uint8 = unsigned char; + +#if TD_MSVC +#pragma warning(push) +#pragma warning(disable : 4309) +#endif + +static_assert(static_cast<char>(128) == -128 || static_cast<char>(128) == 128, + "Unexpected cast to char implementation-defined behaviour"); +static_assert(static_cast<char>(256) == 0, "Unexpected cast to char implementation-defined behaviour"); +static_assert(static_cast<char>(-256) == 0, "Unexpected cast to char implementation-defined behaviour"); + +#if TD_MSVC +#pragma warning(pop) +#endif + +template <size_t size> +struct UInt { + static_assert(size % 8 == 0, "size should be divisible by 8"); + uint8 raw[size / 8]; +}; + +template <size_t size> +inline bool operator==(const UInt<size> &a, const UInt<size> &b) { + return std::memcmp(a.raw, b.raw, sizeof(a.raw)) == 0; +} + +template <size_t size> +inline bool operator!=(const UInt<size> &a, const UInt<size> &b) { + return !(a == b); +} + +using UInt128 = UInt<128>; +using UInt256 = UInt<256>; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/invoke.h b/libs/tdlib/td/tdutils/td/utils/invoke.h new file mode 100644 index 0000000000..e9e56fc2c5 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/invoke.h @@ -0,0 +1,178 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +#include <functional> +#include <tuple> +#include <type_traits> +#include <utility> + +namespace td { + +namespace detail { + +template <std::size_t... S> +struct IntSeq {}; + +template <std::size_t L, std::size_t N, std::size_t... S> +struct IntSeqGen : IntSeqGen<L, N - 1, L + N - 1, S...> {}; + +template <std::size_t L, std::size_t... S> +struct IntSeqGen<L, 0, S...> { + using type = IntSeq<S...>; +}; + +template <bool... Args> +class LogicAndImpl {}; + +template <bool Res, bool X, bool... Args> +class LogicAndImpl<Res, X, Args...> { + public: + static constexpr bool value = LogicAndImpl<(Res && X), Args...>::value; +}; + +template <bool Res> +class LogicAndImpl<Res> { + public: + static constexpr bool value = Res; +}; + +template <std::size_t N> +using IntRange = typename IntSeqGen<0, N>::type; + +template <class T> +struct is_reference_wrapper : std::false_type {}; + +template <class U> +struct is_reference_wrapper<std::reference_wrapper<U>> : std::true_type {}; + +template <class Base, class T, class Derived, class... Args> +auto invoke_impl(T Base::*pmf, Derived &&ref, + Args &&... args) noexcept(noexcept((std::forward<Derived>(ref).*pmf)(std::forward<Args>(args)...))) + -> std::enable_if_t<std::is_function<T>::value && std::is_base_of<Base, std::decay<Derived>>::value, + decltype((std::forward<Derived>(ref).*pmf)(std::forward<Args>(args)...))> { + return (std::forward<Derived>(ref).*pmf)(std::forward<Args>(args)...); +} + +template <class Base, class T, class RefWrap, class... Args> +auto invoke_impl(T Base::*pmf, RefWrap &&ref, + Args &&... args) noexcept(noexcept((ref.get().*pmf)(std::forward<Args>(args)...))) + -> std::enable_if_t<std::is_function<T>::value && is_reference_wrapper<std::decay_t<RefWrap>>::value, + decltype((ref.get().*pmf)(std::forward<Args>(args)...))> + +{ + return (ref.get().*pmf)(std::forward<Args>(args)...); +} + +template <class Base, class T, class Pointer, class... Args> +auto invoke_impl(T Base::*pmf, Pointer &&ptr, + Args &&... args) noexcept(noexcept(((*std::forward<Pointer>(ptr)).*pmf)(std::forward<Args>(args)...))) + -> std::enable_if_t<std::is_function<T>::value && !is_reference_wrapper<std::decay_t<Pointer>>::value && + !std::is_base_of<Base, std::decay_t<Pointer>>::value, + decltype(((*std::forward<Pointer>(ptr)).*pmf)(std::forward<Args>(args)...))> { + return ((*std::forward<Pointer>(ptr)).*pmf)(std::forward<Args>(args)...); +} + +template <class Base, class T, class Derived> +auto invoke_impl(T Base::*pmd, Derived &&ref) noexcept(noexcept(std::forward<Derived>(ref).*pmd)) + -> std::enable_if_t<!std::is_function<T>::value && std::is_base_of<Base, std::decay_t<Derived>>::value, + decltype(std::forward<Derived>(ref).*pmd)> { + return std::forward<Derived>(ref).*pmd; +} + +template <class Base, class T, class RefWrap> +auto invoke_impl(T Base::*pmd, RefWrap &&ref) noexcept(noexcept(ref.get().*pmd)) + -> std::enable_if_t<!std::is_function<T>::value && is_reference_wrapper<std::decay_t<RefWrap>>::value, + decltype(ref.get().*pmd)> { + return ref.get().*pmd; +} + +template <class Base, class T, class Pointer> +auto invoke_impl(T Base::*pmd, Pointer &&ptr) noexcept(noexcept((*std::forward<Pointer>(ptr)).*pmd)) + -> std::enable_if_t<!std::is_function<T>::value && !is_reference_wrapper<std::decay_t<Pointer>>::value && + !std::is_base_of<Base, std::decay_t<Pointer>>::value, + decltype((*std::forward<Pointer>(ptr)).*pmd)> { + return (*std::forward<Pointer>(ptr)).*pmd; +} + +template <class F, class... Args> +auto invoke_impl(F &&f, Args &&... args) noexcept(noexcept(std::forward<F>(f)(std::forward<Args>(args)...))) + -> std::enable_if_t<!std::is_member_pointer<std::decay_t<F>>::value, + decltype(std::forward<F>(f)(std::forward<Args>(args)...))> { + return std::forward<F>(f)(std::forward<Args>(args)...); +} + +template <class F, class... ArgTypes> +auto invoke(F &&f, + ArgTypes &&... args) noexcept(noexcept(invoke_impl(std::forward<F>(f), std::forward<ArgTypes>(args)...))) + -> decltype(invoke_impl(std::forward<F>(f), std::forward<ArgTypes>(args)...)) { + return invoke_impl(std::forward<F>(f), std::forward<ArgTypes>(args)...); +} + +template <class F, class... Args, std::size_t... S> +void call_tuple_impl(F &func, std::tuple<Args...> &&tuple, IntSeq<S...>) { + func(std::forward<Args>(std::get<S>(tuple))...); +} + +template <class... Args, std::size_t... S> +void invoke_tuple_impl(std::tuple<Args...> &&tuple, IntSeq<S...>) { + invoke(std::forward<Args>(std::get<S>(tuple))...); +} + +template <class Actor, class F, class... Args, std::size_t... S> +void mem_call_tuple_impl(Actor *actor, F &func, std::tuple<Args...> &&tuple, IntSeq<S...>) { + (actor->*func)(std::forward<Args>(std::get<S>(tuple))...); +} + +template <class F, class... Args, std::size_t... S> +void tuple_for_each_impl(std::tuple<Args...> &tuple, const F &func, IntSeq<S...>) { + const auto &dummy = {0, (func(std::get<S>(tuple)), 0)...}; + (void)dummy; +} + +template <class F, class... Args, std::size_t... S> +void tuple_for_each_impl(const std::tuple<Args...> &tuple, const F &func, IntSeq<S...>) { + const auto &dummy = {0, (func(std::get<S>(tuple)), 0)...}; + (void)dummy; +} + +} // namespace detail + +template <bool... Args> +class LogicAnd { + public: + static constexpr bool value = detail::LogicAndImpl<true, Args...>::value; +}; + +template <class F, class... Args> +void call_tuple(F &func, std::tuple<Args...> &&tuple) { + detail::call_tuple_impl(func, std::move(tuple), detail::IntRange<sizeof...(Args)>()); +} + +template <class... Args> +void invoke_tuple(std::tuple<Args...> &&tuple) { + detail::invoke_tuple_impl(std::move(tuple), detail::IntRange<sizeof...(Args)>()); +} + +template <class Actor, class F, class... Args> +void mem_call_tuple(Actor *actor, F &func, std::tuple<Args...> &&tuple) { + detail::mem_call_tuple_impl(actor, func, std::move(tuple), detail::IntRange<sizeof...(Args)>()); +} + +template <class F, class... Args> +void tuple_for_each(std::tuple<Args...> &tuple, const F &func) { + detail::tuple_for_each_impl(tuple, func, detail::IntRange<sizeof...(Args)>()); +} + +template <class F, class... Args> +void tuple_for_each(const std::tuple<Args...> &tuple, const F &func) { + detail::tuple_for_each_impl(tuple, func, detail::IntRange<sizeof...(Args)>()); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/logging.cpp b/libs/tdlib/td/tdutils/td/utils/logging.cpp new file mode 100644 index 0000000000..17403ff87b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/logging.cpp @@ -0,0 +1,238 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/logging.h" + +#include "td/utils/port/Clocks.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/Slice.h" +#include "td/utils/Time.h" + +#include <atomic> +#include <cstdlib> + +#if TD_ANDROID +#include <android/log.h> +#define ALOG_TAG "DLTD" +#elif TD_TIZEN +#include <dlog.h> +#define DLOG_TAG "DLTD" +#elif TD_EMSCRIPTEN +#include <emscripten.h> +#endif + +namespace td { + +int VERBOSITY_NAME(level) = VERBOSITY_NAME(DEBUG) + 1; +int VERBOSITY_NAME(net_query) = VERBOSITY_NAME(INFO); +int VERBOSITY_NAME(td_requests) = VERBOSITY_NAME(INFO); +int VERBOSITY_NAME(dc) = VERBOSITY_NAME(DEBUG) + 2; +int VERBOSITY_NAME(files) = VERBOSITY_NAME(DEBUG) + 2; +int VERBOSITY_NAME(mtproto) = VERBOSITY_NAME(DEBUG) + 7; +int VERBOSITY_NAME(connections) = VERBOSITY_NAME(DEBUG) + 8; +int VERBOSITY_NAME(raw_mtproto) = VERBOSITY_NAME(DEBUG) + 10; +int VERBOSITY_NAME(fd) = VERBOSITY_NAME(DEBUG) + 9; +int VERBOSITY_NAME(actor) = VERBOSITY_NAME(DEBUG) + 10; +int VERBOSITY_NAME(buffer) = VERBOSITY_NAME(DEBUG) + 10; +int VERBOSITY_NAME(sqlite) = VERBOSITY_NAME(DEBUG) + 10; + +TD_THREAD_LOCAL const char *Logger::tag_ = nullptr; +TD_THREAD_LOCAL const char *Logger::tag2_ = nullptr; + +Logger::Logger(LogInterface &log, int log_level, Slice file_name, int line_num, Slice comment, bool simple_mode) + : Logger(log, log_level, simple_mode) { + if (simple_mode) { + return; + } + + auto last_slash_ = static_cast<int32>(file_name.size()) - 1; + while (last_slash_ >= 0 && file_name[last_slash_] != '/' && file_name[last_slash_] != '\\') { + last_slash_--; + } + file_name = file_name.substr(last_slash_ + 1); + + auto thread_id = get_thread_id(); + + (*this) << '['; + if (log_level < 10) { + (*this) << ' '; + } + (*this) << log_level << "][t"; + if (thread_id < 10) { + (*this) << ' '; + } + (*this) << thread_id << "][" << StringBuilder::FixedDouble(Clocks::system(), 9) << "][" << file_name << ':' + << line_num << ']'; + if (tag_ != nullptr && *tag_) { + (*this) << "[#" << Slice(tag_) << "]"; + } + if (tag2_ != nullptr && *tag2_) { + (*this) << "[!" << Slice(tag2_) << "]"; + } + if (!comment.empty()) { + (*this) << "[&" << comment << "]"; + } + (*this) << "\t"; +} + +Logger::~Logger() { + if (!simple_mode_) { + sb_ << '\n'; + auto slice = as_cslice(); + if (slice.back() != '\n') { + slice.back() = '\n'; + } + } + + log_.append(as_cslice(), log_level_); +} + +TsCerr::TsCerr() { + enterCritical(); +} +TsCerr::~TsCerr() { + exitCritical(); +} +TsCerr &TsCerr::operator<<(Slice slice) { + auto &fd = Fd::Stderr(); + if (fd.empty()) { + return *this; + } + double end_time = 0; + while (!slice.empty()) { + auto res = fd.write(slice); + if (res.is_error()) { + if (res.error().code() == EPIPE) { + break; + } + // Resource temporary unavailable + if (end_time == 0) { + end_time = Time::now() + 0.01; + } else if (Time::now() > end_time) { + break; + } + continue; + } + slice.remove_prefix(res.ok()); + } + return *this; +} + +void TsCerr::enterCritical() { + while (lock_.test_and_set(std::memory_order_acquire)) { + // spin + } +} + +void TsCerr::exitCritical() { + lock_.clear(std::memory_order_release); +} +TsCerr::Lock TsCerr::lock_ = ATOMIC_FLAG_INIT; + +class DefaultLog : public LogInterface { + public: + void append(CSlice slice, int log_level) override { +#if TD_ANDROID + switch (log_level) { + case VERBOSITY_NAME(FATAL): + __android_log_write(ANDROID_LOG_FATAL, ALOG_TAG, slice.c_str()); + break; + case VERBOSITY_NAME(ERROR): + __android_log_write(ANDROID_LOG_ERROR, ALOG_TAG, slice.c_str()); + break; + case VERBOSITY_NAME(WARNING): + __android_log_write(ANDROID_LOG_WARN, ALOG_TAG, slice.c_str()); + break; + case VERBOSITY_NAME(INFO): + __android_log_write(ANDROID_LOG_INFO, ALOG_TAG, slice.c_str()); + break; + default: + __android_log_write(ANDROID_LOG_DEBUG, ALOG_TAG, slice.c_str()); + break; + } +#elif TD_TIZEN + switch (log_level) { + case VERBOSITY_NAME(FATAL): + dlog_print(DLOG_ERROR, DLOG_TAG, slice.c_str()); + break; + case VERBOSITY_NAME(ERROR): + dlog_print(DLOG_ERROR, DLOG_TAG, slice.c_str()); + break; + case VERBOSITY_NAME(WARNING): + dlog_print(DLOG_WARN, DLOG_TAG, slice.c_str()); + break; + case VERBOSITY_NAME(INFO): + dlog_print(DLOG_INFO, DLOG_TAG, slice.c_str()); + break; + default: + dlog_print(DLOG_DEBUG, DLOG_TAG, slice.c_str()); + break; + } +#elif TD_EMSCRIPTEN + switch (log_level) { + case VERBOSITY_NAME(FATAL): + emscripten_log( + EM_LOG_ERROR | EM_LOG_CONSOLE | EM_LOG_C_STACK | EM_LOG_JS_STACK | EM_LOG_DEMANGLE | EM_LOG_FUNC_PARAMS, + "%s", slice.c_str()); + EM_ASM(throw(UTF8ToString($0)), slice.c_str()); + break; + case VERBOSITY_NAME(ERROR): + emscripten_log(EM_LOG_ERROR | EM_LOG_CONSOLE, "%s", slice.c_str()); + break; + case VERBOSITY_NAME(WARNING): + emscripten_log(EM_LOG_WARN | EM_LOG_CONSOLE, "%s", slice.c_str()); + break; + default: + emscripten_log(EM_LOG_CONSOLE, "%s", slice.c_str()); + break; + } +#elif !TD_WINDOWS + Slice color; + switch (log_level) { + case VERBOSITY_NAME(FATAL): + case VERBOSITY_NAME(ERROR): + color = TC_RED; + break; + case VERBOSITY_NAME(WARNING): + color = TC_YELLOW; + break; + case VERBOSITY_NAME(INFO): + color = TC_CYAN; + break; + } + TsCerr() << color << slice << TC_EMPTY; +#else + // TODO: color + TsCerr() << slice; +#endif + if (log_level == VERBOSITY_NAME(FATAL)) { + process_fatal_error(slice); + } + } + void rotate() override { + } +}; +static DefaultLog default_log; + +LogInterface *const default_log_interface = &default_log; +LogInterface *log_interface = default_log_interface; + +static OnFatalErrorCallback on_fatal_error_callback = nullptr; + +void set_log_fatal_error_callback(OnFatalErrorCallback callback) { + on_fatal_error_callback = callback; +} + +void process_fatal_error(CSlice message) { + auto callback = on_fatal_error_callback; + if (callback) { + callback(message); + } + std::abort(); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/logging.h b/libs/tdlib/td/tdutils/td/utils/logging.h new file mode 100644 index 0000000000..629a4f248a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/logging.h @@ -0,0 +1,279 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +/* + * Simple logging. + * + * Predefined log levels: FATAL, ERROR, WARNING, INFO, DEBUG + * + * LOG(WARNING) << "Hello world!"; + * LOG(INFO) << "Hello " << 1234 << " world!"; + * LOG_IF(INFO, condition) << "Hello world if condition!"; + * + * Custom log levels may be defined and used using VLOG: + * int VERBOSITY_NAME(custom) = VERBOSITY_NAME(WARNING); + * VLOG(custom) << "Hello custom world!" + * + * LOG(FATAL) << "Power is off"; + * CHECK(condition) <===> LOG_IF(FATAL, !(condition)) + */ + +#include "td/utils/common.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/Slice-decl.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/StringBuilder.h" + +#include <atomic> +#include <type_traits> + +#define PSTR_IMPL() ::td::Logger(::td::NullLog().ref(), 0, true) +#define PSLICE() ::td::detail::Slicify() & PSTR_IMPL() +#define PSTRING() ::td::detail::Stringify() & PSTR_IMPL() +#define PSLICE_SAFE() ::td::detail::SlicifySafe() & PSTR_IMPL() +#define PSTRING_SAFE() ::td::detail::StringifySafe() & PSTR_IMPL() + +#define VERBOSITY_NAME(x) verbosity_##x + +#define GET_VERBOSITY_LEVEL() (::td::VERBOSITY_NAME(level)) +#define SET_VERBOSITY_LEVEL(new_level) (::td::VERBOSITY_NAME(level) = (new_level)) + +#ifndef STRIP_LOG +#define STRIP_LOG VERBOSITY_NAME(DEBUG) +#endif +#define LOG_IS_STRIPPED(strip_level) \ + (std::integral_constant<int, VERBOSITY_NAME(strip_level)>() > std::integral_constant<int, STRIP_LOG>()) + +#define LOGGER(level, comment) \ + ::td::Logger(*::td::log_interface, VERBOSITY_NAME(level), __FILE__, __LINE__, comment, \ + VERBOSITY_NAME(level) == VERBOSITY_NAME(PLAIN)) + +#define LOG_IMPL(strip_level, level, condition, comment) \ + LOG_IS_STRIPPED(strip_level) || VERBOSITY_NAME(level) > GET_VERBOSITY_LEVEL() || !(condition) \ + ? (void)0 \ + : ::td::detail::Voidify() & LOGGER(level, comment) + +#define LOG(level) LOG_IMPL(level, level, true, ::td::Slice()) +#define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition) + +#define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level)) +#define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition) + +#define LOG_ROTATE() ::td::log_interface->rotate() + +#define LOG_TAG ::td::Logger::tag_ +#define LOG_TAG2 ::td::Logger::tag2_ + +#if TD_CLANG +bool no_return_func() __attribute__((analyzer_noreturn)); +#endif + +inline bool no_return_func() { + return true; +} + +// clang-format off +#ifdef CHECK + #undef CHECK +#endif +#ifdef TD_DEBUG + #if TD_MSVC + #define CHECK(condition) \ + __analysis_assume(!!(condition)); \ + LOG_IMPL(FATAL, FATAL, !(condition), #condition) + #else + #define CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition) + #endif +#else + #define CHECK(condition) LOG_IF(NEVER, !(condition)) +#endif +// clang-format on + +#define UNREACHABLE() \ + LOG(FATAL); \ + ::td::process_fatal_error("Unreachable in " __FILE__ " at " TD_DEFINE_STR(__LINE__)) + +constexpr int VERBOSITY_NAME(PLAIN) = -1; +constexpr int VERBOSITY_NAME(FATAL) = 0; +constexpr int VERBOSITY_NAME(ERROR) = 1; +constexpr int VERBOSITY_NAME(WARNING) = 2; +constexpr int VERBOSITY_NAME(INFO) = 3; +constexpr int VERBOSITY_NAME(DEBUG) = 4; +constexpr int VERBOSITY_NAME(NEVER) = 1024; + +namespace td { +extern int VERBOSITY_NAME(level); +// TODO Not part of utils. Should be in some separate file +extern int VERBOSITY_NAME(mtproto); +extern int VERBOSITY_NAME(raw_mtproto); +extern int VERBOSITY_NAME(connections); +extern int VERBOSITY_NAME(dc); +extern int VERBOSITY_NAME(fd); +extern int VERBOSITY_NAME(net_query); +extern int VERBOSITY_NAME(td_requests); +extern int VERBOSITY_NAME(actor); +extern int VERBOSITY_NAME(buffer); +extern int VERBOSITY_NAME(files); +extern int VERBOSITY_NAME(sqlite); + +class LogInterface { + public: + LogInterface() = default; + LogInterface(const LogInterface &) = delete; + LogInterface &operator=(const LogInterface &) = delete; + LogInterface(LogInterface &&) = delete; + LogInterface &operator=(LogInterface &&) = delete; + virtual ~LogInterface() = default; + virtual void append(CSlice slice, int log_level_) = 0; + virtual void rotate() = 0; +}; + +class NullLog : public LogInterface { + public: + void append(CSlice slice, int log_level_) override { + } + void rotate() override { + } + NullLog &ref() { + return *this; + } +}; + +extern LogInterface *const default_log_interface; +extern LogInterface *log_interface; + +using OnFatalErrorCallback = void (*)(CSlice message); +void set_log_fatal_error_callback(OnFatalErrorCallback callback); + +[[noreturn]] void process_fatal_error(CSlice message); + +#define TC_RED "\e[1;31m" +#define TC_BLUE "\e[1;34m" +#define TC_CYAN "\e[1;36m" +#define TC_GREEN "\e[1;32m" +#define TC_YELLOW "\e[1;33m" +#define TC_EMPTY "\e[0m" + +class TsCerr { + public: + TsCerr(); + TsCerr(const TsCerr &) = delete; + TsCerr &operator=(const TsCerr &) = delete; + TsCerr(TsCerr &&) = delete; + TsCerr &operator=(TsCerr &&) = delete; + ~TsCerr(); + TsCerr &operator<<(Slice slice); + + private: + using Lock = std::atomic_flag; + static Lock lock_; + + void enterCritical(); + void exitCritical(); +}; + +class Logger { + public: + static const int BUFFER_SIZE = 128 * 1024; + Logger(LogInterface &log, int log_level, bool simple_mode = false) + : buffer_(StackAllocator::alloc(BUFFER_SIZE)) + , log_(log) + , log_level_(log_level) + , sb_(buffer_.as_slice()) + , simple_mode_(simple_mode) { + } + + Logger(LogInterface &log, int log_level, Slice file_name, int line_num, Slice comment, bool simple_mode); + + template <class T> + Logger &operator<<(const T &other) { + sb_ << other; + return *this; + } + + MutableCSlice as_cslice() { + return sb_.as_cslice(); + } + bool is_error() const { + return sb_.is_error(); + } + Logger(const Logger &) = delete; + Logger &operator=(const Logger &) = delete; + Logger(Logger &&) = delete; + Logger &operator=(Logger &&) = delete; + ~Logger(); + + static TD_THREAD_LOCAL const char *tag_; + static TD_THREAD_LOCAL const char *tag2_; + + private: + decltype(StackAllocator::alloc(0)) buffer_; + LogInterface &log_; + int log_level_; + StringBuilder sb_; + bool simple_mode_; +}; + +namespace detail { +class Voidify { + public: + template <class T> + void operator&(const T &) { + } +}; + +class Slicify { + public: + CSlice operator&(Logger &logger) { + return logger.as_cslice(); + } +}; + +class Stringify { + public: + string operator&(Logger &logger) { + return logger.as_cslice().str(); + } +}; +} // namespace detail + +class TsLog : public LogInterface { + public: + explicit TsLog(LogInterface *log) : log_(log) { + } + void init(LogInterface *log) { + enter_critical(); + log_ = log; + exit_critical(); + } + void append(CSlice slice, int level) override { + enter_critical(); + log_->append(slice, level); + exit_critical(); + } + void rotate() override { + enter_critical(); + log_->rotate(); + exit_critical(); + } + + private: + LogInterface *log_ = nullptr; + std::atomic_flag lock_ = ATOMIC_FLAG_INIT; + void enter_critical() { + while (lock_.test_and_set(std::memory_order_acquire)) { + // spin + } + } + void exit_critical() { + lock_.clear(std::memory_order_release); + } +}; +} // namespace td + +#include "td/utils/Slice.h" diff --git a/libs/tdlib/td/tdutils/td/utils/misc.cpp b/libs/tdlib/td/tdutils/td/utils/misc.cpp new file mode 100644 index 0000000000..f3068ca6d3 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/misc.cpp @@ -0,0 +1,78 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/misc.h" + +#include "td/utils/port/thread_local.h" + +#include <algorithm> +#include <cstdlib> +#include <locale> +#include <sstream> + +namespace td { + +char *str_dup(Slice str) { + char *res = static_cast<char *>(std::malloc(str.size() + 1)); + if (res == nullptr) { + return nullptr; + } + std::copy(str.begin(), str.end(), res); + res[str.size()] = '\0'; + return res; +} + +string implode(vector<string> v, char delimiter) { + string result; + for (auto &str : v) { + if (!result.empty()) { + result += delimiter; + } + result += str; + } + return result; +} + +string oneline(Slice str) { + string result; + result.reserve(str.size()); + bool after_new_line = true; + for (auto c : str) { + if (c != '\n') { + if (after_new_line) { + if (c == ' ') { + continue; + } + after_new_line = false; + } + result += c; + } else { + after_new_line = true; + result += ' '; + } + } + while (!result.empty() && result.back() == ' ') { + result.pop_back(); + } + return result; +} + +double to_double(Slice str) { + static TD_THREAD_LOCAL std::stringstream *ss; + if (init_thread_local<std::stringstream>(ss)) { + ss->imbue(std::locale::classic()); + } else { + ss->str(std::string()); + ss->clear(); + } + ss->write(str.begin(), narrow_cast<std::streamsize>(str.size())); + + double result = 0.0; + *ss >> result; + return result; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/misc.h b/libs/tdlib/td/tdutils/td/utils/misc.h new file mode 100644 index 0000000000..62b01794ab --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/misc.h @@ -0,0 +1,337 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include <cstdint> +#include <limits> +#include <tuple> +#include <type_traits> +#include <utility> + +namespace td { + +char *str_dup(Slice str); + +template <class T> +std::pair<T, T> split(T s, char delimiter = ' ') { + auto delimiter_pos = s.find(delimiter); + if (delimiter_pos == string::npos) { + return {std::move(s), T()}; + } else { + return {s.substr(0, delimiter_pos), s.substr(delimiter_pos + 1)}; + } +} + +template <class T> +vector<T> full_split(T s, char delimiter = ' ') { + T next; + vector<T> result; + while (!s.empty()) { + std::tie(next, s) = split(s, delimiter); + result.push_back(next); + } + return result; +} + +string implode(vector<string> v, char delimiter = ' '); + +namespace detail { + +template <typename T> +struct transform_helper { + template <class Func> + auto transform(const T &v, const Func &f) { + vector<decltype(f(*v.begin()))> result; + result.reserve(v.size()); + for (auto &x : v) { + result.push_back(f(x)); + } + return result; + } + + template <class Func> + auto transform(T &&v, const Func &f) { + vector<decltype(f(std::move(*v.begin())))> result; + result.reserve(v.size()); + for (auto &x : v) { + result.push_back(f(std::move(x))); + } + return result; + } +}; + +} // namespace detail + +template <class T, class Func> +auto transform(T &&v, const Func &f) { + return detail::transform_helper<std::decay_t<T>>().transform(std::forward<T>(v), f); +} + +template <class T> +void reset_to_empty(T &value) { + using std::swap; + std::decay_t<T> tmp; + swap(tmp, value); +} + +template <class T> +auto append(vector<T> &destination, const vector<T> &source) { + destination.insert(destination.end(), source.begin(), source.end()); +} + +template <class T> +auto append(vector<T> &destination, vector<T> &&source) { + if (destination.empty()) { + destination.swap(source); + return; + } + destination.reserve(destination.size() + source.size()); + for (auto &elem : source) { + destination.push_back(std::move(elem)); + } + reset_to_empty(source); +} + +inline bool begins_with(Slice str, Slice prefix) { + return prefix.size() <= str.size() && prefix == Slice(str.data(), prefix.size()); +} + +inline bool ends_with(Slice str, Slice suffix) { + return suffix.size() <= str.size() && suffix == Slice(str.data() + str.size() - suffix.size(), suffix.size()); +} + +inline char to_lower(char c) { + if ('A' <= c && c <= 'Z') { + return static_cast<char>(c - 'A' + 'a'); + } + + return c; +} + +inline void to_lower_inplace(MutableSlice slice) { + for (auto &c : slice) { + c = to_lower(c); + } +} + +inline string to_lower(Slice slice) { + auto result = slice.str(); + to_lower_inplace(result); + return result; +} + +inline char to_upper(char c) { + if ('a' <= c && c <= 'z') { + return static_cast<char>(c - 'a' + 'A'); + } + + return c; +} + +inline void to_upper_inplace(MutableSlice slice) { + for (auto &c : slice) { + c = to_upper(c); + } +} + +inline string to_upper(Slice slice) { + auto result = slice.str(); + to_upper_inplace(result); + return result; +} + +inline bool is_space(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\0' || c == '\v'; +} + +inline bool is_alpha(char c) { + c |= 0x20; + return 'a' <= c && c <= 'z'; +} + +inline bool is_digit(char c) { + return '0' <= c && c <= '9'; +} + +inline bool is_alnum(char c) { + return is_alpha(c) || is_digit(c); +} + +inline bool is_hex_digit(char c) { + if (is_digit(c)) { + return true; + } + c |= 0x20; + return 'a' <= c && c <= 'f'; +} + +template <class T> +T trim(T str) { + auto begin = str.data(); + auto end = begin + str.size(); + while (begin < end && is_space(*begin)) { + begin++; + } + while (begin < end && is_space(end[-1])) { + end--; + } + if (static_cast<size_t>(end - begin) == str.size()) { + return std::move(str); + } + return T(begin, end); +} + +string oneline(Slice str); + +template <class T> +std::enable_if_t<std::is_signed<T>::value, T> to_integer(Slice str) { + using unsigned_T = typename std::make_unsigned<T>::type; + unsigned_T integer_value = 0; + auto begin = str.begin(); + auto end = str.end(); + bool is_negative = false; + if (begin != end && *begin == '-') { + is_negative = true; + begin++; + } + while (begin != end && is_digit(*begin)) { + integer_value = static_cast<unsigned_T>(integer_value * 10 + (*begin++ - '0')); + } + if (integer_value > static_cast<unsigned_T>(std::numeric_limits<T>::max())) { + static_assert(~0 + 1 == 0, "Two's complement"); + // Use ~x + 1 instead of -x to suppress Visual Studio warning. + integer_value = static_cast<unsigned_T>(~integer_value + 1); + is_negative = !is_negative; + + if (integer_value > static_cast<unsigned_T>(std::numeric_limits<T>::max())) { + return std::numeric_limits<T>::min(); + } + } + + return is_negative ? static_cast<T>(-static_cast<T>(integer_value)) : static_cast<T>(integer_value); +} + +template <class T> +std::enable_if_t<std::is_unsigned<T>::value, T> to_integer(Slice str) { + T integer_value = 0; + auto begin = str.begin(); + auto end = str.end(); + while (begin != end && is_digit(*begin)) { + integer_value = static_cast<T>(integer_value * 10 + (*begin++ - '0')); + } + return integer_value; +} + +template <class T> +Result<T> to_integer_safe(Slice str) { + auto res = to_integer<T>(str); + if (to_string(res) != str) { + return Status::Error(PSLICE() << "Can't parse \"" << str << "\" as number"); + } + return res; +} + +inline int hex_to_int(char c) { + if (is_digit(c)) { + return c - '0'; + } + c |= 0x20; + if ('a' <= c && c <= 'f') { + return c - 'a' + 10; + } + return 16; +} + +template <class T> +typename std::enable_if<std::is_unsigned<T>::value, T>::type hex_to_integer(Slice str) { + T integer_value = 0; + auto begin = str.begin(); + auto end = str.end(); + while (begin != end && is_hex_digit(*begin)) { + integer_value = static_cast<T>(integer_value * 16 + hex_to_int(*begin++)); + } + return integer_value; +} + +double to_double(Slice str); + +template <class T> +T clamp(T value, T min_value, T max_value) { + if (value < min_value) { + return min_value; + } + if (value > max_value) { + return max_value; + } + return value; +} + +// run-time checked narrowing cast (type conversion): + +namespace detail { +template <class T, class U> +struct is_same_signedness + : public std::integral_constant<bool, std::is_signed<T>::value == std::is_signed<U>::value> {}; + +template <class T, class Enable = void> +struct safe_undeflying_type { + using type = T; +}; + +template <class T> +struct safe_undeflying_type<T, std::enable_if_t<std::is_enum<T>::value>> { + using type = std::underlying_type_t<T>; +}; +} // namespace detail + +template <class R, class A> +R narrow_cast(const A &a) { + using RT = typename detail::safe_undeflying_type<R>::type; + using AT = typename detail::safe_undeflying_type<A>::type; + + static_assert(std::is_integral<RT>::value, "expected integral type to cast to"); + static_assert(std::is_integral<AT>::value, "expected integral type to cast from"); + + auto r = R(a); + CHECK(A(r) == a); + CHECK((detail::is_same_signedness<RT, AT>::value) || ((static_cast<RT>(r) < RT{}) == (static_cast<AT>(a) < AT{}))); + + return r; +} + +template <class R, class A> +Result<R> narrow_cast_safe(const A &a) { + using RT = typename detail::safe_undeflying_type<R>::type; + using AT = typename detail::safe_undeflying_type<A>::type; + + static_assert(std::is_integral<RT>::value, "expected integral type to cast to"); + static_assert(std::is_integral<AT>::value, "expected integral type to cast from"); + + auto r = R(a); + if (!(A(r) == a)) { + return Status::Error("Narrow cast failed"); + } + if (!((detail::is_same_signedness<RT, AT>::value) || ((static_cast<RT>(r) < RT{}) == (static_cast<AT>(a) < AT{})))) { + return Status::Error("Narrow cast failed"); + } + + return r; +} + +template <int Alignment, class T> +bool is_aligned_pointer(const T *pointer) { + static_assert(Alignment > 0 && (Alignment & (Alignment - 1)) == 0, "Wrong alignment"); + return (reinterpret_cast<std::uintptr_t>(static_cast<const void *>(pointer)) & (Alignment - 1)) == 0; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/optional.h b/libs/tdlib/td/tdutils/td/utils/optional.h new file mode 100644 index 0000000000..450b60f94c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/optional.h @@ -0,0 +1,36 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/Status.h" + +#include <utility> + +namespace td { + +template <class T> +class optional { + public: + optional() = default; + template <class T1> + optional(T1 &&t) : impl_(std::forward<T1>(t)) { + } + explicit operator bool() { + return impl_.is_ok(); + } + T &value() { + return impl_.ok_ref(); + } + T &operator*() { + return value(); + } + + private: + Result<T> impl_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/overloaded.h b/libs/tdlib/td/tdutils/td/utils/overloaded.h new file mode 100644 index 0000000000..6c6186f6a2 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/overloaded.h @@ -0,0 +1,39 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +namespace td { + +namespace detail { + +template <class... Fs> +struct overload; + +template <class F> +struct overload<F> : public F { + explicit overload(F f) : F(f) { + } +}; + +template <class F, class... Fs> +struct overload<F, Fs...> + : public overload<F> + , overload<Fs...> { + overload(F f, Fs... fs) : overload<F>(f), overload<Fs...>(fs...) { + } + using overload<F>::operator(); + using overload<Fs...>::operator(); +}; + +} // namespace detail + +template <class... F> +auto overloaded(F... f) { + return detail::overload<F...>(f...); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/Clocks.cpp b/libs/tdlib/td/tdutils/td/utils/port/Clocks.cpp new file mode 100644 index 0000000000..da68754a61 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Clocks.cpp @@ -0,0 +1,23 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/Clocks.h" + +#include <chrono> + +namespace td { + +ClocksDefault::Duration ClocksDefault::monotonic() { + auto duration = std::chrono::steady_clock::now().time_since_epoch(); + return static_cast<double>(std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count()) * 1e-9; +} + +ClocksDefault::Duration ClocksDefault::system() { + auto duration = std::chrono::system_clock::now().time_since_epoch(); + return static_cast<double>(std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count()) * 1e-9; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/Clocks.h b/libs/tdlib/td/tdutils/td/utils/port/Clocks.h new file mode 100644 index 0000000000..a4270df4ad --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Clocks.h @@ -0,0 +1,28 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +namespace td { + +class ClocksBase { + public: + using Duration = double; +}; + +// TODO: (maybe) write system specific functions. +class ClocksDefault { + public: + using Duration = ClocksBase::Duration; + + static Duration monotonic(); + + static Duration system(); +}; + +using Clocks = ClocksDefault; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/CxCli.h b/libs/tdlib/td/tdutils/td/utils/port/CxCli.h new file mode 100644 index 0000000000..b7f01fa401 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/CxCli.h @@ -0,0 +1,133 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#undef small + +#if TD_WINRT + +#include "td/utils/port/wstring_convert.h" + +#include "collection.h" + +#include <cstdint> +#include <map> +#include <mutex> + +#define REF_NEW ref new +#define CLRCALL + +namespace CxCli { + +using Windows::Foundation::Collections::IVector; +#define Array IVector +using Platform::Collections::Vector; +#define ArraySize(arr) ((arr)->Size) +#define ArrayGet(arr, index) ((arr)->GetAt(index)) +#define ArraySet(arr, index, value) ((arr)->SetAt((index), (value))) +#define ArrayIndexType unsigned + +using Platform::String; + +using Platform::NullReferenceException; + +template <class Key, class Value> class ConcurrentDictionary { +public: + bool TryGetValue(Key key, Value &value) { + std::lock_guard<std::mutex> guard(mutex_); + auto it = impl_.find(key); + if (it == impl_.end()) { + return false; + } + value = it->second; + return true; + } + bool TryRemove(Key key, Value &value) { + std::lock_guard<std::mutex> guard(mutex_); + auto it = impl_.find(key); + if (it == impl_.end()) { + return false; + } + value = std::move(it->second); + impl_.erase(it); + return true; + } + Value &operator [] (Key key) { + std::lock_guard<std::mutex> guard(mutex_); + return impl_[key]; + } +private: + std::mutex mutex_; + std::map<Key, Value> impl_; +}; + +inline std::int64_t Increment(volatile std::int64_t &value) { + return InterlockedIncrement64(&value); +} + +inline std::string string_to_unmanaged(String^ str) { + if (!str) { + return std::string(); + } + return td::from_wstring(str->Data(), str->Length()).ok(); +} + +inline String^ string_from_unmanaged(const std::string &from) { + auto tmp = td::to_wstring(from).ok(); + return REF_NEW String(tmp.c_str(), static_cast<unsigned>(tmp.size())); +} + +} // namespace CxCli + +#elif TD_CLI + +#include <msclr\marshal_cppstd.h> + +#define REF_NEW gcnew +#define CLRCALL __clrcall + +namespace CxCli { + +using uint8 = td::uint8; +using int32 = td::int32; +using int64 = td::int64; +using float64 = double; + +#define Array array +#define Vector array +#define ArraySize(arr) ((arr)->Length) +#define ArrayGet(arr, index) ((arr)[index]) +#define ArraySet(arr, index, value) ((arr)[index] = (value)) +#define ArrayIndexType int + +using System::String; + +using System::NullReferenceException; + +using System::Collections::Concurrent::ConcurrentDictionary; + +inline std::int64_t Increment(std::int64_t %value) { + return System::Threading::Interlocked::Increment(value); +} + +inline std::string string_to_unmanaged(String^ str) { + if (!str) { + return std::string(); + } + return msclr::interop::marshal_as<std::string>(str); +} + +inline String^ string_from_unmanaged(const std::string &from) { + return msclr::interop::marshal_as<String^>(from); +} + +} // namespace CxCli + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/EventFd.h b/libs/tdlib/td/tdutils/td/utils/port/EventFd.h new file mode 100644 index 0000000000..ba2edabefd --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/EventFd.h @@ -0,0 +1,33 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +// include all and let config.h decide +#include "td/utils/port/detail/EventFdBsd.h" +#include "td/utils/port/detail/EventFdLinux.h" +#include "td/utils/port/detail/EventFdWindows.h" + +namespace td { + +// clang-format off + +#if TD_EVENTFD_LINUX + using EventFd = detail::EventFdLinux; +#elif TD_EVENTFD_BSD + using EventFd = detail::EventFdBsd; +#elif TD_EVENTFD_WINDOWS + using EventFd = detail::EventFdWindows; +#elif TD_EVENTFD_UNSUPPORTED +#else + #error "EventFd's implementation is not defined" +#endif + +// clang-format on + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/EventFdBase.h b/libs/tdlib/td/tdutils/td/utils/port/EventFdBase.h new file mode 100644 index 0000000000..e119a3c0eb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/EventFdBase.h @@ -0,0 +1,32 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Status.h" + +namespace td { +class EventFdBase { + public: + EventFdBase() = default; + EventFdBase(const EventFdBase &) = delete; + EventFdBase &operator=(const EventFdBase &) = delete; + EventFdBase(EventFdBase &&) = default; + EventFdBase &operator=(EventFdBase &&) = default; + virtual ~EventFdBase() = default; + + virtual void init() = 0; + virtual bool empty() = 0; + virtual void close() = 0; + virtual const Fd &get_fd() const = 0; + virtual Fd &get_fd() = 0; + virtual Status get_pending_error() TD_WARN_UNUSED_RESULT = 0; + virtual void release() = 0; + virtual void acquire() = 0; +}; +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/Fd.cpp b/libs/tdlib/td/tdutils/td/utils/port/Fd.cpp new file mode 100644 index 0000000000..cb4cb27306 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Fd.cpp @@ -0,0 +1,1104 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/Fd.h" + +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Observer.h" + +#if TD_PORT_POSIX + +#include <atomic> + +#include <fcntl.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <unistd.h> + +#endif + +#if TD_PORT_WINDOWS + +#include "td/utils/buffer.h" +#include "td/utils/misc.h" + +#include <cstring> + +#endif + +namespace td { + +#if TD_PORT_POSIX + +Fd::InfoSet::InfoSet() { + get_info(0).refcnt = 1; + get_info(1).refcnt = 1; + get_info(2).refcnt = 1; +} +Fd::Info &Fd::InfoSet::get_info(int32 id) { + CHECK(0 <= id && id < InfoSet::MAX_FD) << tag("fd", id); + return fd_array_[id]; +} +Fd::InfoSet Fd::fd_info_set_; + +// TODO(bug) if constuctor call tries to output something to the LOG it will fail, because log is not initialized +Fd Fd::stderr_(2, Mode::Reference); +Fd Fd::stdout_(1, Mode::Reference); +Fd Fd::stdin_(0, Mode::Reference); + +Fd::Fd() = default; + +Fd::Fd(int fd, Mode mode) : mode_(mode), fd_(fd) { + auto *info = get_info(); + int old_ref_cnt = info->refcnt.load(std::memory_order_relaxed); + if (old_ref_cnt == 0) { + old_ref_cnt = info->refcnt.load(std::memory_order_acquire); + CHECK(old_ref_cnt == 0); + CHECK(mode_ == Mode::Owner) << tag("fd", fd_); + VLOG(fd) << "FD created [fd:" << fd_ << "]"; + + auto fcntl_res = fcntl(fd_, F_GETFD); + auto fcntl_errno = errno; + LOG_IF(FATAL, fcntl_res == -1) << Status::PosixError(fcntl_errno, "fcntl F_GET_FD failed"); + + info->refcnt.store(1, std::memory_order_relaxed); + CHECK(mode_ != Mode::Reference); + CHECK(info->observer == nullptr); + info->flags = 0; + info->observer = nullptr; + } else { + CHECK(mode_ == Mode::Reference) << tag("fd", fd_); + auto fcntl_res = fcntl(fd_, F_GETFD); + auto fcntl_errno = errno; + LOG_IF(FATAL, fcntl_res == -1) << Status::PosixError(fcntl_errno, "fcntl F_GET_FD failed"); + + CHECK(mode_ == Mode::Reference); + info->refcnt.fetch_add(1, std::memory_order_relaxed); + } +} + +int Fd::move_as_native_fd() { + clear_info(); + auto res = fd_; + fd_ = -1; + return res; +} + +Fd::~Fd() { + close(); +} + +Fd::Fd(Fd &&other) { + fd_ = other.fd_; + mode_ = other.mode_; + other.fd_ = -1; +} + +Fd &Fd::operator=(Fd &&other) { + if (this != &other) { + close(); + + fd_ = other.fd_; + mode_ = other.mode_; + other.fd_ = -1; + } + return *this; +} + +Fd Fd::clone() const { + return Fd(fd_, Mode::Reference); +} + +Fd &Fd::Stderr() { + return stderr_; +} +Fd &Fd::Stdout() { + return stdout_; +} +Fd &Fd::Stdin() { + return stdin_; +} + +Status Fd::duplicate(const Fd &from, Fd &to) { + CHECK(!from.empty()); + CHECK(!to.empty()); + if (dup2(from.get_native_fd(), to.get_native_fd()) == -1) { + return OS_ERROR("dup2 failed"); + } + return Status::OK(); +} + +bool Fd::empty() const { + return fd_ == -1; +} + +const Fd &Fd::get_fd() const { + return *this; +} + +Fd &Fd::get_fd() { + return *this; +} + +int Fd::get_native_fd() const { + CHECK(!empty()); + return fd_; +} + +void Fd::set_observer(ObserverBase *observer) { + auto *info = get_info(); + CHECK(observer == nullptr || info->observer == nullptr); + info->observer = observer; +} + +ObserverBase *Fd::get_observer() const { + auto *info = get_info(); + return info->observer; +} + +void Fd::close_ref() { + CHECK(mode_ == Mode::Reference); + auto *info = get_info(); + + int old_ref_cnt = info->refcnt.fetch_sub(1, std::memory_order_relaxed); + CHECK(old_ref_cnt > 1) << tag("fd", fd_); + fd_ = -1; +} + +void Fd::close_own() { + CHECK(mode_ == Mode::Owner); + VLOG(fd) << "FD closed [fd:" << fd_ << "]"; + + clear_info(); + ::close(fd_); + fd_ = -1; +} + +void Fd::close() { + if (!empty()) { + switch (mode_) { + case Mode::Reference: + close_ref(); + break; + case Mode::Owner: + close_own(); + break; + } + } +} + +Fd::Info *Fd::get_info() { + CHECK(!empty()); + return &fd_info_set_.get_info(fd_); +} + +const Fd::Info *Fd::get_info() const { + CHECK(!empty()); + return &fd_info_set_.get_info(fd_); +} + +void Fd::clear_info() { + CHECK(!empty()); + CHECK(mode_ != Mode::Reference); + + auto *info = get_info(); + int old_ref_cnt = info->refcnt.load(std::memory_order_relaxed); + CHECK(old_ref_cnt == 1); + info->flags = 0; + info->observer = nullptr; + info->refcnt.store(0, std::memory_order_release); +} + +void Fd::update_flags_notify(Flags flags) { + update_flags_inner(flags, true); +} + +void Fd::update_flags(Flags flags) { + update_flags_inner(flags, false); +} + +void Fd::update_flags_inner(int32 new_flags, bool notify_flag) { + if (new_flags & Error) { + new_flags |= Error; + new_flags |= Close; + } + auto *info = get_info(); + int32 &flags = info->flags; + int32 old_flags = flags; + flags |= new_flags; + if (new_flags & Close) { + // TODO: ??? + flags &= ~Write; + } + if (flags != old_flags) { + VLOG(fd) << "Update flags " << tag("fd", fd_) << tag("from", format::as_binary(old_flags)) + << tag("to", format::as_binary(flags)); + } + if (flags != old_flags && notify_flag) { + auto observer = info->observer; + if (observer != nullptr) { + observer->notify(); + } + } +} + +Fd::Flags Fd::get_flags() const { + return get_info()->flags; +} + +void Fd::clear_flags(Flags flags) { + get_info()->flags &= ~flags; +} + +bool Fd::has_pending_error() const { + return (get_flags() & Fd::Flag::Error) != 0; +} + +Status Fd::get_pending_error() { + if (!has_pending_error()) { + return Status::OK(); + } + clear_flags(Fd::Error); + int error = 0; + socklen_t errlen = sizeof(error); + if (getsockopt(fd_, SOL_SOCKET, SO_ERROR, static_cast<void *>(&error), &errlen) == 0) { + if (error == 0) { + return Status::OK(); + } + return Status::PosixError(error, PSLICE() << "Error on socket [fd_ = " << fd_ << "]"); + } + auto status = OS_SOCKET_ERROR(PSLICE() << "Can't load error on socket [fd_ = " << fd_ << "]"); + LOG(INFO) << "Can't load pending socket error: " << status; + return status; +} + +Result<size_t> Fd::write_unsafe(Slice slice) { + int native_fd = get_native_fd(); + auto write_res = skip_eintr([&] { return ::write(native_fd, slice.begin(), slice.size()); }); + auto write_errno = errno; + if (write_res >= 0) { + return narrow_cast<size_t>(write_res); + } + return Status::PosixError(write_errno, PSLICE() << "Write to fd " << native_fd << " has failed"); +} + +Result<size_t> Fd::write(Slice slice) { + int native_fd = get_native_fd(); + auto write_res = skip_eintr([&] { return ::write(native_fd, slice.begin(), slice.size()); }); + auto write_errno = errno; + if (write_res >= 0) { + return narrow_cast<size_t>(write_res); + } + + if (write_errno == EAGAIN +#if EAGAIN != EWOULDBLOCK + || write_errno == EWOULDBLOCK +#endif + ) { + clear_flags(Write); + return 0; + } + + auto error = Status::PosixError(write_errno, PSLICE() << "Write to fd " << native_fd << " has failed"); + switch (write_errno) { + case EBADF: + case ENXIO: + case EFAULT: + case EINVAL: + LOG(FATAL) << error; + UNREACHABLE(); + default: + LOG(WARNING) << error; + // fallthrough + case ECONNRESET: + case EDQUOT: + case EFBIG: + case EIO: + case ENETDOWN: + case ENETUNREACH: + case ENOSPC: + case EPIPE: + clear_flags(Write); + update_flags(Close); + return std::move(error); + } +} + +Result<size_t> Fd::read(MutableSlice slice) { + int native_fd = get_native_fd(); + CHECK(slice.size() > 0); + auto read_res = skip_eintr([&] { return ::read(native_fd, slice.begin(), slice.size()); }); + auto read_errno = errno; + if (read_res >= 0) { + if (read_res == 0) { + errno = 0; + clear_flags(Read); + update_flags(Close); + } + return narrow_cast<size_t>(read_res); + } + if (read_errno == EAGAIN +#if EAGAIN != EWOULDBLOCK + || read_errno == EWOULDBLOCK +#endif + ) { + clear_flags(Read); + return 0; + } + auto error = Status::PosixError(read_errno, PSLICE() << "Read from fd " << native_fd << " has failed"); + switch (read_errno) { + case EISDIR: + case EBADF: + case ENXIO: + case EFAULT: + case EINVAL: + case ENOTCONN: + LOG(FATAL) << error; + UNREACHABLE(); + default: + LOG(WARNING) << error; + // fallthrough + case EIO: + case ENOBUFS: + case ENOMEM: + case ECONNRESET: + case ETIMEDOUT: + clear_flags(Read); + update_flags(Close); + return std::move(error); + } +} + +Status Fd::set_is_blocking(bool is_blocking) { + auto old_flags = fcntl(fd_, F_GETFL); + if (old_flags == -1) { + return OS_SOCKET_ERROR("Failed to get socket flags"); + } + auto new_flags = is_blocking ? old_flags & ~O_NONBLOCK : old_flags | O_NONBLOCK; + if (new_flags != old_flags && fcntl(fd_, F_SETFL, new_flags) == -1) { + return OS_SOCKET_ERROR("Failed to set socket flags"); + } + + return Status::OK(); +} + +#endif + +#if TD_PORT_WINDOWS + +class Fd::FdImpl { + public: + FdImpl(Fd::Type type, HANDLE handle) + : type_(type), handle_(handle), async_mode_(type_ == Fd::Type::EventFd || type_ == Fd::Type::StdinFileFd) { + init(); + } + FdImpl(Fd::Type type, SOCKET socket, int socket_family) + : type_(type), socket_(socket), socket_family_(socket_family), async_mode_(true) { + init(); + } + + FdImpl(const FdImpl &) = delete; + FdImpl &operator=(const FdImpl &) = delete; + FdImpl(FdImpl &&) = delete; + FdImpl &operator=(FdImpl &&) = delete; + + ~FdImpl() { + close(); + } + void set_observer(ObserverBase *observer) { + observer_ = observer; + } + ObserverBase *get_observer() const { + return observer_; + } + + void update_flags_notify(Fd::Flags flags) { + update_flags_inner(flags, true); + } + + void update_flags(Fd::Flags flags) { + update_flags_inner(flags, false); + } + + void update_flags_inner(int32 new_flags, bool notify_flag) { + if (new_flags & Fd::Error) { + new_flags |= Fd::Error; + new_flags |= Fd::Close; + } + int32 old_flags = flags_; + flags_ |= new_flags; + if (new_flags & Fd::Close) { + // TODO: ??? + flags_ &= ~Fd::Write; + internal_flags_ &= ~Fd::Write; + } + if (flags_ != old_flags) { + VLOG(fd) << "Update flags " << tag("fd", get_io_handle()) << tag("from", format::as_binary(old_flags)) + << tag("to", format::as_binary(flags_)); + } + if (flags_ != old_flags && notify_flag) { + auto observer = get_observer(); + if (observer != nullptr) { + observer->notify(); + } + } + } + + int32 get_flags() const { + return flags_; + } + + void clear_flags(Fd::Flags mask) { + flags_ &= ~mask; + } + + Status get_pending_error() { + if (!has_pending_error()) { + return Status::OK(); + } + clear_flags(Fd::Error); + return std::move(pending_error_); + } + bool has_pending_error() const { + return (get_flags() & Fd::Flag::Error) != 0; + } + + HANDLE get_read_event() { + if (type() == Fd::Type::StdinFileFd) { + return get_io_handle(); + } + return read_event_; + } + void on_read_event() { + if (type_ == Fd::Type::StdinFileFd) { + return try_read_stdin(); + } + ResetEvent(read_event_); + if (type_ == Fd::Type::EventFd) { + return update_flags_notify(Fd::Flag::Read); + } + if (type_ == Fd::Type::SocketFd && !connected_) { + on_connect_ready(); + } else { + if (!async_read_flag_) { + return; + } + + if (type_ == Fd::Type::ServerSocketFd) { + on_accept_ready(); + } else { + on_read_ready(); + } + } + loop(); + } + HANDLE get_write_event() { + return write_event_; + } + void on_write_event() { + CHECK(async_write_flag_); + ResetEvent(write_event_); + on_write_ready(); + loop(); + } + + SOCKET get_native_socket() const { + return socket_; + } + + HANDLE get_io_handle() const { + CHECK(!empty()); + if (type() == Fd::Type::FileFd || type() == Fd::Type::StdinFileFd) { + return handle_; + } + return reinterpret_cast<HANDLE>(socket_); + } + + Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT { + if (async_mode_) { + return write_async(slice); + } else { + return write_sync(slice); + } + } + + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT { + if (async_mode_) { + return read_async(slice); + } else { + return read_sync(slice); + } + } + + Result<size_t> write_async(Slice slice) TD_WARN_UNUSED_RESULT { + CHECK(async_mode_); + output_writer_.append(slice); + output_reader_.sync_with_writer(); + loop(); + return slice.size(); + } + Result<size_t> write_sync(Slice slice) TD_WARN_UNUSED_RESULT { + CHECK(!async_mode_); + DWORD bytes_written = 0; + auto res = WriteFile(get_io_handle(), slice.data(), narrow_cast<DWORD>(slice.size()), &bytes_written, nullptr); + if (!res) { + return OS_ERROR("Failed to write_sync"); + } + return bytes_written; + } + Result<size_t> read_async(MutableSlice slice) TD_WARN_UNUSED_RESULT { + CHECK(async_mode_); + auto res = input_reader_.advance(min(slice.size(), input_reader_.size()), slice); + if (res == 0) { + clear_flags(Fd::Flag::Read); + } + return res; + } + Result<size_t> read_sync(MutableSlice slice) TD_WARN_UNUSED_RESULT { + CHECK(!async_mode_); + DWORD bytes_read = 0; + auto res = ReadFile(get_io_handle(), slice.data(), narrow_cast<DWORD>(slice.size()), &bytes_read, nullptr); + if (!res) { + return OS_ERROR("Failed to read_sync"); + } + if (bytes_read == 0) { + clear_flags(Fd::Flag::Read); + } + return bytes_read; + } + + // for ServerSocket + Result<Fd> accept() { + if (accepted_.empty()) { + clear_flags(Fd::Flag::Read); + return Status::Error(-1, "Operation would block"); + } + auto res = std::move(accepted_.back()); + accepted_.pop_back(); + return std::move(res); + } + + void connect(const IPAddress &addr) { + CHECK(!connected_); + CHECK(type_ == Fd::Type::SocketFd); + DWORD bytes_read; + std::memset(&read_overlapped_, 0, sizeof(read_overlapped_)); + read_overlapped_.hEvent = read_event_; + LPFN_CONNECTEX ConnectExPtr = nullptr; + GUID guid = WSAID_CONNECTEX; + DWORD numBytes; + int error = ::WSAIoctl(socket_, SIO_GET_EXTENSION_FUNCTION_POINTER, static_cast<void *>(&guid), sizeof(guid), + static_cast<void *>(&ConnectExPtr), sizeof(ConnectExPtr), &numBytes, nullptr, nullptr); + if (error) { + return on_error(OS_SOCKET_ERROR("WSAIoctl failed"), Fd::Flag::Read); + } + auto status = ConnectExPtr(socket_, addr.get_sockaddr(), narrow_cast<int>(addr.get_sockaddr_len()), nullptr, 0, + &bytes_read, &read_overlapped_); + if (status != 0) { + ResetEvent(read_event_); + connected_ = true; + update_flags_notify(Fd::Flag::Read); + return; + } + + auto last_error = GetLastError(); + if (last_error == ERROR_IO_PENDING) { + return; + } + on_error(OS_SOCKET_ERROR("ConnectEx failed"), Fd::Flag::Read); + } + + // for EventFd + void release() { + CHECK(type_ == Fd::Type::EventFd); + SetEvent(read_event_); + } + + void acquire() { + CHECK(type_ == Fd::Type::EventFd); + ResetEvent(read_event_); + clear_flags(Fd::Flag::Read); + } + + // TODO: interface for BufferedFd optimization. + + bool empty() const { + return type() == Fd::Type::Empty; + } + void close() { + if (empty()) { + return; + } + switch (type()) { + case Fd::Type::StdinFileFd: + case Fd::Type::FileFd: { + if (!CloseHandle(handle_)) { + auto error = OS_ERROR("Failed to close file"); + LOG(ERROR) << error; + } + handle_ = INVALID_HANDLE_VALUE; + break; + } + case Fd::Type::ServerSocketFd: + case Fd::Type::SocketFd: { + if (closesocket(socket_) != 0) { + auto error = OS_SOCKET_ERROR("Failed to close socket"); + LOG(ERROR) << error; + } + socket_ = INVALID_SOCKET; + break; + } + case Fd::Type::EventFd: + break; + default: + UNREACHABLE(); + } + + if (read_event_ != INVALID_HANDLE_VALUE) { + if (!CloseHandle(read_event_)) { + auto error = OS_ERROR("Failed to close event"); + LOG(ERROR) << error; + } + read_event_ = INVALID_HANDLE_VALUE; + } + if (write_event_ != INVALID_HANDLE_VALUE) { + if (!CloseHandle(write_event_)) { + auto error = OS_ERROR("Failed to close event"); + LOG(ERROR) << error; + } + write_event_ = INVALID_HANDLE_VALUE; + } + + type_ = Fd::Type::Empty; + } + + private: + Fd::Type type_; + HANDLE handle_ = INVALID_HANDLE_VALUE; + SOCKET socket_ = INVALID_SOCKET; + + int socket_family_ = 0; + + bool async_mode_ = false; + + ObserverBase *observer_ = nullptr; + Fd::Flags flags_ = Fd::Flag::Write; + Status pending_error_; + Fd::Flags internal_flags_ = Fd::Flag::Write | Fd::Flag::Read; + + HANDLE read_event_ = INVALID_HANDLE_VALUE; // used by WineventPoll + bool async_read_flag_ = false; // do we have pending read? + OVERLAPPED read_overlapped_; + ChainBufferWriter input_writer_; + ChainBufferReader input_reader_ = input_writer_.extract_reader(); + + bool connected_ = false; + + std::vector<Fd> accepted_; + SOCKET accept_socket_ = INVALID_SOCKET; + static constexpr size_t MAX_ADDR_SIZE = sizeof(sockaddr_in6) + 16; + char addr_buf_[MAX_ADDR_SIZE * 2]; + + HANDLE write_event_ = INVALID_HANDLE_VALUE; // used by WineventPoll + bool async_write_flag_ = false; // do we have pending write? + OVERLAPPED write_overlapped_; + ChainBufferWriter output_writer_; + ChainBufferReader output_reader_ = output_writer_.extract_reader(); + + void init() { + if (async_mode_) { + if (type_ != Fd::Type::EventFd) { + write_event_ = CreateEventW(nullptr, true, false, nullptr); + } + read_event_ = CreateEventW(nullptr, true, false, nullptr); + loop(); + } + } + + Fd::Type type() const { + return type_; + } + + void on_error(Status error, Fd::Flag flag) { + VLOG(fd) << tag("fd", get_io_handle()) << error; + pending_error_ = std::move(error); + internal_flags_ &= ~flag; + update_flags_notify(Fd::Flag::Error); + } + void on_eof() { + internal_flags_ &= ~Fd::Flag::Read; + update_flags_notify(Fd::Flag::Close); + } + + void on_read_ready() { + async_read_flag_ = false; + DWORD bytes_read; + auto status = GetOverlappedResult(get_io_handle(), &read_overlapped_, &bytes_read, false); + if (status == 0) { + return on_error(OS_ERROR("ReadFile failed"), Fd::Flag::Read); + } + + VLOG(fd) << "Read " << tag("fd", get_io_handle()) << tag("size", bytes_read); + if (bytes_read == 0) { // eof + return on_eof(); + } + input_writer_.confirm_append(bytes_read); + input_reader_.sync_with_writer(); + update_flags_notify(Fd::Flag::Read); + } + void on_write_ready() { + async_write_flag_ = false; + DWORD bytes_written; + auto status = GetOverlappedResult(get_io_handle(), &write_overlapped_, &bytes_written, false); + if (status == 0) { + return on_error(OS_ERROR("WriteFile failed"), Fd::Flag::Write); + } + if (bytes_written != 0) { + VLOG(fd) << "Write " << tag("fd", get_io_handle()) << tag("size", bytes_written); + output_reader_.confirm_read(bytes_written); + update_flags_notify(Fd::Flag::Write); + } + } + + void on_accept_ready() { + async_read_flag_ = false; + DWORD bytes_read; + auto status = GetOverlappedResult(get_io_handle(), &read_overlapped_, &bytes_read, false); + if (status == 0) { + return on_error(OS_ERROR("AcceptEx failed"), Fd::Flag::Write); + } + accepted_.push_back(Fd::create_socket_fd(accept_socket_)); + accept_socket_ = INVALID_SOCKET; + update_flags_notify(Fd::Flag::Read); + } + + void on_connect_ready() { + async_read_flag_ = false; + DWORD bytes_read; + VLOG(fd) << "on_connect_ready"; + auto status = GetOverlappedResult(get_io_handle(), &read_overlapped_, &bytes_read, false); + if (status == 0) { + return on_error(OS_ERROR("ConnectEx failed"), Fd::Flag::Write); + } + connected_ = true; + VLOG(fd) << "connected = true"; + } + + void try_read_stdin() { + } + void try_start_read() { + auto dest = input_writer_.prepare_append(); + DWORD bytes_read; + std::memset(&read_overlapped_, 0, sizeof(read_overlapped_)); + read_overlapped_.hEvent = read_event_; + VLOG(fd) << "try_read.."; + auto status = + ReadFile(get_io_handle(), dest.data(), narrow_cast<DWORD>(dest.size()), &bytes_read, &read_overlapped_); + if (status != 0) { // ok + ResetEvent(read_event_); + VLOG(fd) << "Read " << tag("fd", get_io_handle()) << tag("size", bytes_read); + if (bytes_read == 0) { // eof + return on_eof(); + } + input_writer_.confirm_append(bytes_read); + input_reader_.sync_with_writer(); + update_flags_notify(Fd::Flag::Read); + return; + } + auto last_error = GetLastError(); + if (last_error == ERROR_IO_PENDING) { + async_read_flag_ = true; + return; + } + on_error(OS_ERROR("ReadFile failed"), Fd::Flag::Read); + } + + void try_start_write() { + auto dest = output_reader_.prepare_read(); + DWORD bytes_written; + std::memset(&write_overlapped_, 0, sizeof(write_overlapped_)); + write_overlapped_.hEvent = write_event_; + VLOG(fd) << "try_start_write"; + auto status = + WriteFile(get_io_handle(), dest.data(), narrow_cast<DWORD>(dest.size()), &bytes_written, &write_overlapped_); + if (status != 0) { // ok + VLOG(fd) << "Write " << tag("fd", get_io_handle()) << tag("size", bytes_written); + ResetEvent(write_event_); + output_reader_.confirm_read(bytes_written); + update_flags_notify(Fd::Flag::Write); + return; + } + auto last_error = GetLastError(); + if (last_error == ERROR_IO_PENDING) { + VLOG(fd) << "try_start_write: ERROR_IO_PENDING"; + async_write_flag_ = true; + return; + } + CHECK(WaitForSingleObject(write_event_, 0) != WAIT_OBJECT_0); + on_error(OS_ERROR("WriteFile failed"), Fd::Flag::Write); + } + + void try_start_accept() { + if (async_read_flag_ == true) { + return; + } + accept_socket_ = socket(socket_family_, SOCK_STREAM, 0); + DWORD bytes_read; + std::memset(&read_overlapped_, 0, sizeof(read_overlapped_)); + read_overlapped_.hEvent = read_event_; + auto status = + AcceptEx(socket_, accept_socket_, addr_buf_, 0, MAX_ADDR_SIZE, MAX_ADDR_SIZE, &bytes_read, &read_overlapped_); + if (status != 0) { + ResetEvent(read_event_); + accepted_.push_back(Fd::create_socket_fd(accept_socket_)); + accept_socket_ = INVALID_SOCKET; + update_flags_notify(Fd::Flag::Read); + return; + } + + auto last_error = GetLastError(); + if (last_error == ERROR_IO_PENDING) { + async_read_flag_ = true; + return; + } + on_error(OS_SOCKET_ERROR("AcceptExFailed"), Fd::Flag::Read); + } + + void loop() { + CHECK(async_mode_); + + if (type_ == Fd::Type::EventFd) { + return; + } + if (type_ == Fd::Type::ServerSocketFd) { + while (async_read_flag_ == false && (internal_flags_ & Fd::Flag::Read) != 0) { + // read always + try_start_accept(); + } + return; + } + if (!connected_) { + return; + } + while (async_read_flag_ == false && (internal_flags_ & Fd::Flag::Read) != 0) { + // read always + try_start_read(); + } + VLOG(fd) << (async_write_flag_ == false) << " " << output_reader_.size() << " " + << ((internal_flags_ & Fd::Flag::Write) != 0); + while (async_write_flag_ == false && output_reader_.size() && (internal_flags_ & Fd::Flag::Write) != 0) { + // write if we have data to write + try_start_write(); + } + } +}; + +Fd::Fd() = default; + +Fd::Fd(Fd &&other) = default; + +Fd &Fd::operator=(Fd &&other) = default; + +Fd::~Fd() = default; + +Fd Fd::create_file_fd(HANDLE handle) { + return Fd(Fd::Type::FileFd, Fd::Mode::Owner, handle); +} + +Fd Fd::create_socket_fd(SOCKET sock) { + return Fd(Fd::Type::SocketFd, Fd::Mode::Owner, sock, AF_UNSPEC); +} + +Fd Fd::create_server_socket_fd(SOCKET sock, int socket_family) { + return Fd(Fd::Type::ServerSocketFd, Fd::Mode::Owner, sock, socket_family); +} + +Fd Fd::create_event_fd() { + return Fd(Fd::Type::EventFd, Fd::Mode::Owner, INVALID_HANDLE_VALUE); +} + +const Fd &Fd::get_fd() const { + return *this; +} + +Fd &Fd::get_fd() { + return *this; +} + +Result<size_t> Fd::read(MutableSlice slice) { + return impl_->read(slice); +} + +Result<size_t> Fd::write(Slice slice) { + CHECK(!empty()); + return impl_->write(slice); +} + +bool Fd::empty() const { + return !impl_; +} + +void Fd::close() { + impl_.reset(); +} + +Result<Fd> Fd::accept() { + return impl_->accept(); +} +void Fd::connect(const IPAddress &addr) { + return impl_->connect(addr); +} + +Fd Fd::clone() const { + return Fd(impl_); +} + +uint64 Fd::get_key() const { + return reinterpret_cast<uint64>(impl_.get()); +} + +void Fd::set_observer(ObserverBase *observer) { + return impl_->set_observer(observer); +} +ObserverBase *Fd::get_observer() const { + return impl_->get_observer(); +} + +Fd::Flags Fd::get_flags() const { + return impl_->get_flags(); +} +void Fd::update_flags(Flags flags) { + impl_->update_flags(flags); +} + +void Fd::on_read_event() { + impl_->on_read_event(); +} +void Fd::on_write_event() { + impl_->on_write_event(); +} + +bool Fd::has_pending_error() const { + return impl_->has_pending_error(); +} +Status Fd::get_pending_error() { + return impl_->get_pending_error(); +} + +HANDLE Fd::get_read_event() { + return impl_->get_read_event(); +} +HANDLE Fd::get_write_event() { + return impl_->get_write_event(); +} + +SOCKET Fd::get_native_socket() const { + return impl_->get_native_socket(); +} + +HANDLE Fd::get_io_handle() const { + return impl_->get_io_handle(); +} + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) +Fd &Fd::Stderr() { + static auto handle = GetStdHandle(STD_ERROR_HANDLE); + LOG_IF(FATAL, handle == INVALID_HANDLE_VALUE) << "Failed to get stderr"; + static auto fd = Fd(Fd::Type::FileFd, Fd::Mode::Reference, handle); + return fd; +} +Fd &Fd::Stdin() { + static auto handle = GetStdHandle(STD_INPUT_HANDLE); + LOG_IF(FATAL, handle == INVALID_HANDLE_VALUE) << "Failed to get stdin"; + static auto fd = Fd(Fd::Type::FileFd, Fd::Mode::Reference, handle); + return fd; +} +Fd &Fd::Stdout() { + static auto handle = GetStdHandle(STD_OUTPUT_HANDLE); + LOG_IF(FATAL, handle == INVALID_HANDLE_VALUE) << "Failed to get stdout"; + static auto fd = Fd(Fd::Type::FileFd, Fd::Mode::Reference, handle); + return fd; +} +#else +Fd &Fd::Stderr() { + static Fd result; + result = Fd(); + return result; +} +Fd &Fd::Stdin() { + static Fd result; + result = Fd(); + return result; +} +Fd &Fd::Stdout() { + static Fd result; + result = Fd(); + return result; +} +#endif + +Status Fd::duplicate(const Fd &from, Fd &to) { + return Status::Error("Not supported"); +} + +Status Fd::set_is_blocking(bool is_blocking) { + return detail::set_native_socket_is_blocking(get_native_socket(), is_blocking); +} + +Fd::Fd(Type type, Mode mode, HANDLE handle) : mode_(mode), impl_(std::make_shared<FdImpl>(type, handle)) { +} + +Fd::Fd(Type type, Mode mode, SOCKET sock, int socket_family) + : mode_(mode), impl_(std::make_shared<FdImpl>(type, sock, socket_family)) { +} + +Fd::Fd(std::shared_ptr<FdImpl> impl) : mode_(Mode::Reference), impl_(std::move(impl)) { +} + +void Fd::acquire() { + return impl_->acquire(); +} + +void Fd::release() { + return impl_->release(); +} + +class InitWSA { + public: + InitWSA() { + /* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */ + WORD wVersionRequested = MAKEWORD(2, 2); + WSADATA wsaData; + if (WSAStartup(wVersionRequested, &wsaData) != 0) { + auto error = OS_SOCKET_ERROR("Failed to init WSA"); + LOG(FATAL) << error; + } + } +}; + +static InitWSA init_wsa; + +#endif + +namespace detail { +#if TD_PORT_POSIX +Status set_native_socket_is_blocking(int fd, bool is_blocking) { + if (fcntl(fd, F_SETFL, is_blocking ? 0 : O_NONBLOCK) == -1) { +#elif TD_PORT_WINDOWS +Status set_native_socket_is_blocking(SOCKET fd, bool is_blocking) { + u_long mode = is_blocking; + if (ioctlsocket(fd, FIONBIO, &mode) != 0) { +#endif + return OS_SOCKET_ERROR("Failed to change socket flags"); + } + return Status::OK(); +} +} // namespace detail + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/Fd.h b/libs/tdlib/td/tdutils/td/utils/port/Fd.h new file mode 100644 index 0000000000..5ce9b6cedc --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Fd.h @@ -0,0 +1,226 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#if TD_PORT_WINDOWS +#include "td/utils/port/IPAddress.h" + +#include <memory> +#endif + +#if TD_PORT_POSIX +#include <errno.h> + +#include <atomic> +#include <type_traits> +#endif + +namespace td { + +class ObserverBase; + +#if TD_PORT_WINDOWS +namespace detail { +class EventFdWindows; +} // namespace detail +#endif + +class Fd { + public: + // TODO: Close may be not enough + // Sometimes descriptor is half-closed. + + enum Flag : int32 { + Write = 0x001, + Read = 0x002, + Close = 0x004, + Error = 0x008, + All = Write | Read | Close | Error, + None = 0 + }; + using Flags = int32; + enum class Mode { Reference, Owner }; + + Fd(); + Fd(const Fd &) = delete; + Fd &operator=(const Fd &) = delete; + Fd(Fd &&other); + Fd &operator=(Fd &&other); + ~Fd(); + +#if TD_PORT_POSIX + Fd(int fd, Mode mode); +#endif +#if TD_PORT_WINDOWS + static Fd create_file_fd(HANDLE handle); + + static Fd create_socket_fd(SOCKET sock); + + static Fd create_server_socket_fd(SOCKET sock, int socket_family); + + static Fd create_event_fd(); +#endif + + Fd clone() const; + + static Fd &Stderr(); + static Fd &Stdout(); + static Fd &Stdin(); + + static Status duplicate(const Fd &from, Fd &to); + + bool empty() const; + + const Fd &get_fd() const; + Fd &get_fd(); + + void set_observer(ObserverBase *observer); + ObserverBase *get_observer() const; + + void close(); + + void update_flags(Flags flags); + + Flags get_flags() const; + + bool has_pending_error() const; + Status get_pending_error() TD_WARN_UNUSED_RESULT; + + Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT; + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT; + + Status set_is_blocking(bool is_blocking); + +#if TD_PORT_POSIX + void update_flags_notify(Flags flags); + void clear_flags(Flags flags); + + Result<size_t> write_unsafe(Slice slice) TD_WARN_UNUSED_RESULT; + + int get_native_fd() const; + int move_as_native_fd(); +#endif + +#if TD_PORT_WINDOWS + Result<Fd> accept() TD_WARN_UNUSED_RESULT; + void connect(const IPAddress &addr); + + uint64 get_key() const; + + HANDLE get_read_event(); + HANDLE get_write_event(); + void on_read_event(); + void on_write_event(); + + SOCKET get_native_socket() const; + HANDLE get_io_handle() const; +#endif + + private: + Mode mode_ = Mode::Owner; + +#if TD_PORT_POSIX + struct Info { + std::atomic<int> refcnt; + int32 flags; + ObserverBase *observer; + }; + struct InfoSet { + InfoSet(); + Info &get_info(int32 id); + + private: + static constexpr int MAX_FD = 1 << 18; + Info fd_array_[MAX_FD]; + }; + static InfoSet fd_info_set_; + + static Fd stderr_; + static Fd stdout_; + static Fd stdin_; + + void update_flags_inner(int32 new_flags, bool notify_flag); + Info *get_info(); + const Info *get_info() const; + void clear_info(); + + void close_ref(); + void close_own(); + + int fd_ = -1; +#endif +#if TD_PORT_WINDOWS + class FdImpl; + + enum class Type { Empty, EventFd, FileFd, StdinFileFd, SocketFd, ServerSocketFd }; + + Fd(Type type, Mode mode, HANDLE handle); + Fd(Type type, Mode mode, SOCKET sock, int socket_family); + explicit Fd(std::shared_ptr<FdImpl> impl); + + friend class detail::EventFdWindows; // for release and acquire + + void acquire(); + void release(); + + std::shared_ptr<FdImpl> impl_; +#endif +}; + +#if TD_PORT_POSIX +template <class F> +auto skip_eintr(F &&f) { + decltype(f()) res; + static_assert(std::is_integral<decltype(res)>::value, "integral type expected"); + do { + errno = 0; // just in case + res = f(); + } while (res < 0 && errno == EINTR); + return res; +} +template <class F> +auto skip_eintr_cstr(F &&f) { + char *res; + do { + errno = 0; // just in case + res = f(); + } while (res == nullptr && errno == EINTR); + return res; +} +#endif + +template <class FdT> +bool can_read(const FdT &fd) { + return (fd.get_flags() & Fd::Read) != 0; +} + +template <class FdT> +bool can_write(const FdT &fd) { + return (fd.get_flags() & Fd::Write) != 0; +} + +template <class FdT> +bool can_close(const FdT &fd) { + return (fd.get_flags() & Fd::Close) != 0; +} + +namespace detail { +#if TD_PORT_POSIX +Status set_native_socket_is_blocking(int fd, bool is_blocking); +#endif +#if TD_PORT_WINDOWS +Status set_native_socket_is_blocking(SOCKET fd, bool is_blocking); +#endif +} // namespace detail + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/FileFd.cpp b/libs/tdlib/td/tdutils/td/utils/port/FileFd.cpp new file mode 100644 index 0000000000..17a2727f64 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/FileFd.cpp @@ -0,0 +1,481 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/FileFd.h" + +#if TD_PORT_WINDOWS +#include "td/utils/misc.h" // for narrow_cast + +#include "td/utils/port/Stat.h" +#include "td/utils/port/wstring_convert.h" +#endif + +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/sleep.h" +#include "td/utils/StringBuilder.h" + +#include <cstring> + +#if TD_PORT_POSIX +#include <fcntl.h> +#include <sys/file.h> +#include <sys/types.h> +#include <sys/uio.h> +#include <unistd.h> +#endif + +namespace td { + +namespace { + +struct PrintFlags { + int32 flags; +}; + +StringBuilder &operator<<(StringBuilder &sb, const PrintFlags &print_flags) { + auto flags = print_flags.flags; + if (flags & + ~(FileFd::Write | FileFd::Read | FileFd::Truncate | FileFd::Create | FileFd::Append | FileFd::CreateNew)) { + return sb << "opened with invalid flags " << flags; + } + + if (flags & FileFd::Create) { + sb << "opened/created "; + } else if (flags & FileFd::CreateNew) { + sb << "created "; + } else { + sb << "opened "; + } + + if ((flags & FileFd::Write) && (flags & FileFd::Read)) { + if (flags & FileFd::Append) { + sb << "for reading and appending"; + } else { + sb << "for reading and writing"; + } + } else if (flags & FileFd::Write) { + if (flags & FileFd::Append) { + sb << "for appending"; + } else { + sb << "for writing"; + } + } else if (flags & FileFd::Read) { + sb << "for reading"; + } else { + sb << "for nothing"; + } + + if (flags & FileFd::Truncate) { + sb << " with truncation"; + } + return sb; +} + +} // namespace + +const Fd &FileFd::get_fd() const { + return fd_; +} + +Fd &FileFd::get_fd() { + return fd_; +} + +Result<FileFd> FileFd::open(CSlice filepath, int32 flags, int32 mode) { + if (flags & ~(Write | Read | Truncate | Create | Append | CreateNew)) { + return Status::Error(PSLICE() << "File \"" << filepath << "\" has failed to be " << PrintFlags{flags}); + } + + if ((flags & (Write | Read)) == 0) { + return Status::Error(PSLICE() << "File \"" << filepath << "\" can't be " << PrintFlags{flags}); + } + +#if TD_PORT_POSIX + int native_flags = 0; + + if ((flags & Write) && (flags & Read)) { + native_flags |= O_RDWR; + } else if (flags & Write) { + native_flags |= O_WRONLY; + } else { + CHECK(flags & Read); + native_flags |= O_RDONLY; + } + + if (flags & Truncate) { + native_flags |= O_TRUNC; + } + + if (flags & Create) { + native_flags |= O_CREAT; + } else if (flags & CreateNew) { + native_flags |= O_CREAT; + native_flags |= O_EXCL; + } + + if (flags & Append) { + native_flags |= O_APPEND; + } + + int native_fd = skip_eintr([&] { return ::open(filepath.c_str(), native_flags, static_cast<mode_t>(mode)); }); + if (native_fd < 0) { + return OS_ERROR(PSLICE() << "File \"" << filepath << "\" can't be " << PrintFlags{flags}); + } + + FileFd result; + result.fd_ = Fd(native_fd, Fd::Mode::Owner); +#elif TD_PORT_WINDOWS + // TODO: support modes + auto r_filepath = to_wstring(filepath); + if (r_filepath.is_error()) { + return Status::Error(PSLICE() << "Failed to convert file path \" << filepath << \" to UTF-16"); + } + auto w_filepath = r_filepath.move_as_ok(); + DWORD desired_access = 0; + if ((flags & Write) && (flags & Read)) { + desired_access |= GENERIC_READ | GENERIC_WRITE; + } else if (flags & Write) { + desired_access |= GENERIC_WRITE; + } else { + CHECK(flags & Read); + desired_access |= GENERIC_READ; + } + + // TODO: share mode + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE | FILE_SHARE_WRITE; + + DWORD creation_disposition = 0; + if (flags & Create) { + if (flags & Truncate) { + creation_disposition = CREATE_ALWAYS; + } else { + creation_disposition = OPEN_ALWAYS; + } + } else if (flags & CreateNew) { + creation_disposition = CREATE_NEW; + } else { + if (flags & Truncate) { + creation_disposition = TRUNCATE_EXISTING; + } else { + creation_disposition = OPEN_EXISTING; + } + } + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) + auto handle = CreateFile(w_filepath.c_str(), desired_access, share_mode, nullptr, creation_disposition, 0, nullptr); +#else + auto handle = CreateFile2(w_filepath.c_str(), desired_access, share_mode, creation_disposition, nullptr); +#endif + if (handle == INVALID_HANDLE_VALUE) { + return OS_ERROR(PSLICE() << "File \"" << filepath << "\" can't be " << PrintFlags{flags}); + } + if (flags & Append) { + LARGE_INTEGER offset; + offset.QuadPart = 0; + auto set_pointer_res = SetFilePointerEx(handle, offset, nullptr, FILE_END); + if (!set_pointer_res) { + auto res = OS_ERROR(PSLICE() << "Failed to seek to the end of file \"" << filepath << "\""); + CloseHandle(handle); + return res; + } + } + FileFd result; + result.fd_ = Fd::create_file_fd(handle); +#endif + result.fd_.update_flags(Fd::Flag::Write); + return std::move(result); +} + +Result<size_t> FileFd::write(Slice slice) { +#if TD_PORT_POSIX + CHECK(!fd_.empty()); + int native_fd = get_native_fd(); + auto write_res = skip_eintr([&] { return ::write(native_fd, slice.begin(), slice.size()); }); + if (write_res >= 0) { + return narrow_cast<size_t>(write_res); + } + + auto write_errno = errno; + auto error = Status::PosixError(write_errno, PSLICE() << "Write to [fd = " << native_fd << "] has failed"); + if (write_errno != EAGAIN +#if EAGAIN != EWOULDBLOCK + && write_errno != EWOULDBLOCK +#endif + && write_errno != EIO) { + LOG(ERROR) << error; + } + return std::move(error); +#elif TD_PORT_WINDOWS + return fd_.write(slice); +#endif +} + +Result<size_t> FileFd::read(MutableSlice slice) { +#if TD_PORT_POSIX + CHECK(!fd_.empty()); + int native_fd = get_native_fd(); + auto read_res = skip_eintr([&] { return ::read(native_fd, slice.begin(), slice.size()); }); + auto read_errno = errno; + + if (read_res >= 0) { + if (narrow_cast<size_t>(read_res) < slice.size()) { + fd_.clear_flags(Read); + } + return static_cast<size_t>(read_res); + } + + auto error = Status::PosixError(read_errno, PSLICE() << "Read from [fd = " << native_fd << "] has failed"); + if (read_errno != EAGAIN +#if EAGAIN != EWOULDBLOCK + && read_errno != EWOULDBLOCK +#endif + && read_errno != EIO) { + LOG(ERROR) << error; + } + return std::move(error); +#elif TD_PORT_WINDOWS + return fd_.read(slice); +#endif +} + +Result<size_t> FileFd::pwrite(Slice slice, int64 offset) { + if (offset < 0) { + return Status::Error("Offset must be non-negative"); + } +#if TD_PORT_POSIX + TRY_RESULT(offset_off_t, narrow_cast_safe<off_t>(offset)); + CHECK(!fd_.empty()); + int native_fd = get_native_fd(); + auto pwrite_res = skip_eintr([&] { return ::pwrite(native_fd, slice.begin(), slice.size(), offset_off_t); }); + if (pwrite_res >= 0) { + return narrow_cast<size_t>(pwrite_res); + } + + auto pwrite_errno = errno; + auto error = Status::PosixError( + pwrite_errno, PSLICE() << "Pwrite to [fd = " << native_fd << "] at [offset = " << offset << "] has failed"); + if (pwrite_errno != EAGAIN +#if EAGAIN != EWOULDBLOCK + && pwrite_errno != EWOULDBLOCK +#endif + && pwrite_errno != EIO) { + LOG(ERROR) << error; + } + return std::move(error); +#elif TD_PORT_WINDOWS + DWORD bytes_written = 0; + OVERLAPPED overlapped; + std::memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = static_cast<DWORD>(offset); + overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32); + auto res = + WriteFile(fd_.get_io_handle(), slice.data(), narrow_cast<DWORD>(slice.size()), &bytes_written, &overlapped); + if (!res) { + return OS_ERROR("Failed to pwrite"); + } + return bytes_written; +#endif +} + +Result<size_t> FileFd::pread(MutableSlice slice, int64 offset) { + if (offset < 0) { + return Status::Error("Offset must be non-negative"); + } +#if TD_PORT_POSIX + TRY_RESULT(offset_off_t, narrow_cast_safe<off_t>(offset)); + CHECK(!fd_.empty()); + int native_fd = get_native_fd(); + auto pread_res = skip_eintr([&] { return ::pread(native_fd, slice.begin(), slice.size(), offset_off_t); }); + if (pread_res >= 0) { + return narrow_cast<size_t>(pread_res); + } + + auto pread_errno = errno; + auto error = Status::PosixError( + pread_errno, PSLICE() << "Pread from [fd = " << native_fd << "] at [offset = " << offset << "] has failed"); + if (pread_errno != EAGAIN +#if EAGAIN != EWOULDBLOCK + && pread_errno != EWOULDBLOCK +#endif + && pread_errno != EIO) { + LOG(ERROR) << error; + } + return std::move(error); +#elif TD_PORT_WINDOWS + DWORD bytes_read = 0; + OVERLAPPED overlapped; + std::memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = static_cast<DWORD>(offset); + overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32); + auto res = ReadFile(fd_.get_io_handle(), slice.data(), narrow_cast<DWORD>(slice.size()), &bytes_read, &overlapped); + if (!res) { + return OS_ERROR("Failed to pread"); + } + return bytes_read; +#endif +} + +Status FileFd::lock(FileFd::LockFlags flags, int32 max_tries) { + if (max_tries <= 0) { + return Status::Error(0, "Can't lock file: wrong max_tries"); + } + + while (true) { +#if TD_PORT_POSIX + struct flock lock; + std::memset(&lock, 0, sizeof(lock)); + + lock.l_type = static_cast<short>([&] { + switch (flags) { + case LockFlags::Read: + return F_RDLCK; + case LockFlags::Write: + return F_WRLCK; + case LockFlags::Unlock: + return F_UNLCK; + default: + UNREACHABLE(); + return F_UNLCK; + } + }()); + + lock.l_whence = SEEK_SET; + if (fcntl(get_native_fd(), F_SETLK, &lock) == -1) { + if (errno == EAGAIN && --max_tries > 0) { +#elif TD_PORT_WINDOWS + OVERLAPPED overlapped; + std::memset(&overlapped, 0, sizeof(overlapped)); + + BOOL result; + if (flags == LockFlags::Unlock) { + result = UnlockFileEx(fd_.get_io_handle(), 0, MAXDWORD, MAXDWORD, &overlapped); + } else { + DWORD dw_flags = LOCKFILE_FAIL_IMMEDIATELY; + if (flags == LockFlags::Write) { + dw_flags |= LOCKFILE_EXCLUSIVE_LOCK; + } + + result = LockFileEx(fd_.get_io_handle(), dw_flags, 0, MAXDWORD, MAXDWORD, &overlapped); + } + + if (!result) { + if (GetLastError() == ERROR_LOCK_VIOLATION && --max_tries > 0) { +#endif + usleep_for(100000); + continue; + } + + return OS_ERROR("Can't lock file"); + } + return Status::OK(); + } +} + +void FileFd::close() { + fd_.close(); +} + +bool FileFd::empty() const { + return fd_.empty(); +} + +#if TD_PORT_POSIX +int FileFd::get_native_fd() const { + return fd_.get_native_fd(); +} +#endif + +int32 FileFd::get_flags() const { + return fd_.get_flags(); +} + +void FileFd::update_flags(Fd::Flags mask) { + fd_.update_flags(mask); +} + +int64 FileFd::get_size() { + return stat().size_; +} + +#if TD_PORT_WINDOWS +static uint64 filetime_to_unix_time_nsec(LONGLONG filetime) { + const auto FILETIME_UNIX_TIME_DIFF = 116444736000000000ll; + return static_cast<uint64>((filetime - FILETIME_UNIX_TIME_DIFF) * 100); +} +#endif + +Stat FileFd::stat() { + CHECK(!empty()); +#if TD_PORT_POSIX + return detail::fstat(get_native_fd()); +#elif TD_PORT_WINDOWS + Stat res; + + FILE_BASIC_INFO basic_info; + auto status = GetFileInformationByHandleEx(fd_.get_io_handle(), FileBasicInfo, &basic_info, sizeof(basic_info)); + if (!status) { + auto error = OS_ERROR("Stat failed"); + LOG(FATAL) << error; + } + res.atime_nsec_ = filetime_to_unix_time_nsec(basic_info.LastAccessTime.QuadPart); + res.mtime_nsec_ = filetime_to_unix_time_nsec(basic_info.LastWriteTime.QuadPart); + res.is_dir_ = (basic_info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + res.is_reg_ = true; + + FILE_STANDARD_INFO standard_info; + status = GetFileInformationByHandleEx(fd_.get_io_handle(), FileStandardInfo, &standard_info, sizeof(standard_info)); + if (!status) { + auto error = OS_ERROR("Stat failed"); + LOG(FATAL) << error; + } + res.size_ = standard_info.EndOfFile.QuadPart; + + return res; +#endif +} + +Status FileFd::sync() { + CHECK(!empty()); +#if TD_PORT_POSIX + if (fsync(fd_.get_native_fd()) != 0) { +#elif TD_PORT_WINDOWS + if (FlushFileBuffers(fd_.get_io_handle()) == 0) { +#endif + return OS_ERROR("Sync failed"); + } + return Status::OK(); +} + +Status FileFd::seek(int64 position) { + CHECK(!empty()); +#if TD_PORT_POSIX + TRY_RESULT(position_off_t, narrow_cast_safe<off_t>(position)); + if (skip_eintr([&] { return ::lseek(fd_.get_native_fd(), position_off_t, SEEK_SET); }) < 0) { +#elif TD_PORT_WINDOWS + LARGE_INTEGER offset; + offset.QuadPart = position; + if (SetFilePointerEx(fd_.get_io_handle(), offset, nullptr, FILE_BEGIN) == 0) { +#endif + return OS_ERROR("Seek failed"); + } + return Status::OK(); +} + +Status FileFd::truncate_to_current_position(int64 current_position) { + CHECK(!empty()); +#if TD_PORT_POSIX + TRY_RESULT(current_position_off_t, narrow_cast_safe<off_t>(current_position)); + if (skip_eintr([&] { return ::ftruncate(fd_.get_native_fd(), current_position_off_t); }) < 0) { +#elif TD_PORT_WINDOWS + if (SetEndOfFile(fd_.get_io_handle()) == 0) { +#endif + return OS_ERROR("Truncate failed"); + } + return Status::OK(); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/FileFd.h b/libs/tdlib/td/tdutils/td/utils/port/FileFd.h new file mode 100644 index 0000000000..bf7166c1de --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/FileFd.h @@ -0,0 +1,63 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/Stat.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +class FileFd { + public: + FileFd() = default; + + enum Flags : int32 { Write = 1, Read = 2, Truncate = 4, Create = 8, Append = 16, CreateNew = 32 }; + + const Fd &get_fd() const; + Fd &get_fd(); + + static Result<FileFd> open(CSlice filepath, int32 flags, int32 mode = 0600) TD_WARN_UNUSED_RESULT; + + Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT; + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT; + + Result<size_t> pwrite(Slice slice, int64 offset) TD_WARN_UNUSED_RESULT; + Result<size_t> pread(MutableSlice slice, int64 offset) TD_WARN_UNUSED_RESULT; + + enum class LockFlags { Write, Read, Unlock }; + Status lock(LockFlags flags, int32 max_tries = 1) TD_WARN_UNUSED_RESULT; + + void close(); + bool empty() const; + + int32 get_flags() const; + void update_flags(Fd::Flags mask); + + int64 get_size(); + + Stat stat(); + + Status sync() TD_WARN_UNUSED_RESULT; + + Status seek(int64 position) TD_WARN_UNUSED_RESULT; + + Status truncate_to_current_position(int64 current_position) TD_WARN_UNUSED_RESULT; + +#if TD_PORT_POSIX + int get_native_fd() const; +#endif + + private: + Fd fd_; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/IPAddress.cpp b/libs/tdlib/td/tdutils/td/utils/port/IPAddress.cpp new file mode 100644 index 0000000000..2d3a3cdbc0 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/IPAddress.cpp @@ -0,0 +1,361 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/IPAddress.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/port/thread_local.h" +#include "td/utils/ScopeGuard.h" + +#if !TD_WINDOWS +#include <netdb.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <sys/socket.h> +#include <sys/types.h> +#endif + +#include <cstring> + +namespace td { + +IPAddress::IPAddress() : is_valid_(false) { +} + +bool IPAddress::is_valid() const { + return is_valid_; +} + +const sockaddr *IPAddress::get_sockaddr() const { + return &sockaddr_; +} + +size_t IPAddress::get_sockaddr_len() const { + CHECK(is_valid()); + switch (addr_.ss_family) { + case AF_INET6: + return sizeof(ipv6_addr_); + case AF_INET: + return sizeof(ipv4_addr_); + default: + LOG(FATAL) << "Unknown address family"; + return 0; + } +} + +int IPAddress::get_address_family() const { + return get_sockaddr()->sa_family; +} + +bool IPAddress::is_ipv4() const { + return get_address_family() == AF_INET; +} + +uint32 IPAddress::get_ipv4() const { + CHECK(is_valid()); + CHECK(is_ipv4()); + return ipv4_addr_.sin_addr.s_addr; +} + +Slice IPAddress::get_ipv6() const { + static_assert(sizeof(ipv6_addr_.sin6_addr) == 16, "ipv6 size == 16"); + CHECK(is_valid()); + CHECK(!is_ipv4()); + return Slice(ipv6_addr_.sin6_addr.s6_addr, 16); +} + +IPAddress IPAddress::get_any_addr() const { + IPAddress res; + switch (get_address_family()) { + case AF_INET6: + res.init_ipv6_any(); + break; + case AF_INET: + res.init_ipv4_any(); + break; + default: + LOG(FATAL) << "Unknown address family"; + } + return res; +} +void IPAddress::init_ipv4_any() { + is_valid_ = true; + ipv4_addr_.sin_family = AF_INET; + ipv4_addr_.sin_addr.s_addr = INADDR_ANY; + ipv4_addr_.sin_port = 0; +} +void IPAddress::init_ipv6_any() { + is_valid_ = true; + ipv6_addr_.sin6_family = AF_INET6; + ipv6_addr_.sin6_addr = in6addr_any; + ipv6_addr_.sin6_port = 0; +} + +Status IPAddress::init_ipv6_port(CSlice ipv6, int port) { + is_valid_ = false; + if (port <= 0 || port >= (1 << 16)) { + return Status::Error(PSLICE() << "Invalid [port=" << port << "]"); + } + std::memset(&ipv6_addr_, 0, sizeof(ipv6_addr_)); + ipv6_addr_.sin6_family = AF_INET6; + ipv6_addr_.sin6_port = htons(static_cast<uint16>(port)); + int err = inet_pton(AF_INET6, ipv6.c_str(), &ipv6_addr_.sin6_addr); + if (err == 0) { + return Status::Error(PSLICE() << "Failed inet_pton(AF_INET6, " << ipv6 << ")"); + } else if (err == -1) { + return OS_SOCKET_ERROR(PSLICE() << "Failed inet_pton(AF_INET6, " << ipv6 << ")"); + } + is_valid_ = true; + return Status::OK(); +} + +Status IPAddress::init_ipv6_as_ipv4_port(CSlice ipv4, int port) { + return init_ipv6_port(string("::FFFF:").append(ipv4.begin(), ipv4.size()), port); +} + +Status IPAddress::init_ipv4_port(CSlice ipv4, int port) { + is_valid_ = false; + if (port <= 0 || port >= (1 << 16)) { + return Status::Error(PSLICE() << "Invalid [port=" << port << "]"); + } + std::memset(&ipv4_addr_, 0, sizeof(ipv4_addr_)); + ipv4_addr_.sin_family = AF_INET; + ipv4_addr_.sin_port = htons(static_cast<uint16>(port)); + int err = inet_pton(AF_INET, ipv4.c_str(), &ipv4_addr_.sin_addr); + if (err == 0) { + return Status::Error(PSLICE() << "Failed inet_pton(AF_INET, " << ipv4 << ")"); + } else if (err == -1) { + return OS_SOCKET_ERROR(PSLICE() << "Failed inet_pton(AF_INET, " << ipv4 << ")"); + } + is_valid_ = true; + return Status::OK(); +} + +Status IPAddress::init_host_port(CSlice host, int port) { + auto str_port = to_string(port); + return init_host_port(host, str_port); +} + +Status IPAddress::init_host_port(CSlice host, CSlice port) { + addrinfo hints; + addrinfo *info = nullptr; + std::memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // TODO AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + LOG(INFO) << "Try to init IP address of " << host << " with port " << port; + auto s = getaddrinfo(host.c_str(), port.c_str(), &hints, &info); + if (s != 0) { + return Status::Error(PSLICE() << "getaddrinfo: " << gai_strerror(s)); + } + SCOPE_EXIT { + freeaddrinfo(info); + }; + + // prefer ipv4 + addrinfo *best_info = info; + for (auto *ptr = info->ai_next; ptr != nullptr; ptr = ptr->ai_next) { + if (ptr->ai_socktype == AF_INET) { + best_info = ptr; + break; + } + } + // just use first address + CHECK(best_info != nullptr); + return init_sockaddr(best_info->ai_addr, narrow_cast<socklen_t>(best_info->ai_addrlen)); +} + +Status IPAddress::init_host_port(CSlice host_port) { + auto pos = host_port.rfind(':'); + if (pos == static_cast<size_t>(-1)) { + return Status::Error("Can't split string into host and port"); + } + return init_host_port(host_port.substr(0, pos).str(), host_port.substr(pos + 1).str()); +} + +Status IPAddress::init_sockaddr(sockaddr *addr, socklen_t len) { + if (addr->sa_family == AF_INET6) { + CHECK(len == sizeof(ipv6_addr_)); + std::memcpy(&ipv6_addr_, reinterpret_cast<sockaddr_in6 *>(addr), sizeof(ipv6_addr_)); + LOG(INFO) << "Have ipv6 address " << get_ip_str() << " with port " << get_port(); + } else if (addr->sa_family == AF_INET) { + CHECK(len == sizeof(ipv4_addr_)); + std::memcpy(&ipv4_addr_, reinterpret_cast<sockaddr_in *>(addr), sizeof(ipv4_addr_)); + LOG(INFO) << "Have ipv4 address " << get_ip_str() << " with port " << get_port(); + } else { + return Status::Error(PSLICE() << "Unknown " << tag("sa_family", addr->sa_family)); + } + + is_valid_ = true; + return Status::OK(); +} + +Status IPAddress::init_socket_address(const SocketFd &socket_fd) { + is_valid_ = false; +#if TD_WINDOWS + auto fd = socket_fd.get_fd().get_native_socket(); +#else + auto fd = socket_fd.get_fd().get_native_fd(); +#endif + socklen_t len = sizeof(addr_); + int ret = getsockname(fd, &sockaddr_, &len); + if (ret != 0) { + return OS_SOCKET_ERROR("Failed to get socket address"); + } + is_valid_ = true; + return Status::OK(); +} + +Status IPAddress::init_peer_address(const SocketFd &socket_fd) { + is_valid_ = false; +#if TD_WINDOWS + auto fd = socket_fd.get_fd().get_native_socket(); +#else + auto fd = socket_fd.get_fd().get_native_fd(); +#endif + socklen_t len = sizeof(addr_); + int ret = getpeername(fd, &sockaddr_, &len); + if (ret != 0) { + return OS_SOCKET_ERROR("Failed to get peer socket address"); + } + is_valid_ = true; + return Status::OK(); +} + +static CSlice get_ip_str(int family, const void *addr) { + const int buf_size = INET6_ADDRSTRLEN; //, INET_ADDRSTRLEN; + static TD_THREAD_LOCAL char *buf; + init_thread_local<char[]>(buf, buf_size); + + const char *res = inet_ntop(family, +#if TD_WINDOWS + const_cast<PVOID>(addr), +#else + addr, +#endif + buf, buf_size); + if (res == nullptr) { + return CSlice(); + } else { + return CSlice(res); + } +} + +CSlice IPAddress::ipv4_to_str(int32 ipv4) { + auto tmp_ipv4 = ntohl(ipv4); + return ::td::get_ip_str(AF_INET, &tmp_ipv4); +} + +Slice IPAddress::get_ip_str() const { + if (!is_valid()) { + return Slice("0.0.0.0"); + } + + const void *addr; + switch (get_address_family()) { + case AF_INET6: + addr = &ipv6_addr_.sin6_addr; + break; + case AF_INET: + addr = &ipv4_addr_.sin_addr; + break; + default: + UNREACHABLE(); + return Slice(); + } + return ::td::get_ip_str(get_address_family(), addr); +} + +int IPAddress::get_port() const { + if (!is_valid()) { + return 0; + } + + switch (get_address_family()) { + case AF_INET6: + return ntohs(ipv6_addr_.sin6_port); + case AF_INET: + return ntohs(ipv4_addr_.sin_port); + default: + UNREACHABLE(); + return 0; + } +} + +void IPAddress::set_port(int port) { + CHECK(is_valid()); + + switch (get_address_family()) { + case AF_INET6: + ipv6_addr_.sin6_port = htons(static_cast<uint16>(port)); + break; + case AF_INET: + ipv4_addr_.sin_port = htons(static_cast<uint16>(port)); + break; + default: + UNREACHABLE(); + } +} + +bool operator==(const IPAddress &a, const IPAddress &b) { + if (!a.is_valid() || !b.is_valid()) { + return false; + } + if (a.get_address_family() != b.get_address_family()) { + return false; + } + + if (a.get_address_family() == AF_INET) { + return a.ipv4_addr_.sin_port == b.ipv4_addr_.sin_port && + std::memcmp(&a.ipv4_addr_.sin_addr, &b.ipv4_addr_.sin_addr, sizeof(a.ipv4_addr_.sin_addr)) == 0; + } else if (a.get_address_family() == AF_INET6) { + return a.ipv6_addr_.sin6_port == b.ipv6_addr_.sin6_port && + std::memcmp(&a.ipv6_addr_.sin6_addr, &b.ipv6_addr_.sin6_addr, sizeof(a.ipv6_addr_.sin6_addr)) == 0; + } + + LOG(FATAL) << "Unknown address family"; + return false; +} + +bool operator<(const IPAddress &a, const IPAddress &b) { + if (a.is_valid() != b.is_valid()) { + return a.is_valid() < b.is_valid(); + } + if (a.get_address_family() != b.get_address_family()) { + return a.get_address_family() < b.get_address_family(); + } + + if (a.get_address_family() == AF_INET) { + if (a.ipv4_addr_.sin_port != b.ipv4_addr_.sin_port) { + return a.ipv4_addr_.sin_port < b.ipv4_addr_.sin_port; + } + return std::memcmp(&a.ipv4_addr_.sin_addr, &b.ipv4_addr_.sin_addr, sizeof(a.ipv4_addr_.sin_addr)) < 0; + } else if (a.get_address_family() == AF_INET6) { + if (a.ipv6_addr_.sin6_port != b.ipv6_addr_.sin6_port) { + return a.ipv6_addr_.sin6_port < b.ipv6_addr_.sin6_port; + } + return std::memcmp(&a.ipv6_addr_.sin6_addr, &b.ipv6_addr_.sin6_addr, sizeof(a.ipv6_addr_.sin6_addr)) < 0; + } + + LOG(FATAL) << "Unknown address family"; + return false; +} + +StringBuilder &operator<<(StringBuilder &builder, const IPAddress &address) { + if (!address.is_valid()) { + return builder << "[invalid]"; + } + if (address.get_address_family() == AF_INET) { + return builder << "[" << address.get_ip_str() << ":" << address.get_port() << "]"; + } else { + CHECK(address.get_address_family() == AF_INET6); + return builder << "[[" << address.get_ip_str() << "]:" << address.get_port() << "]"; + } +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/IPAddress.h b/libs/tdlib/td/tdutils/td/utils/port/IPAddress.h new file mode 100644 index 0000000000..116a4c5425 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/IPAddress.h @@ -0,0 +1,71 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#if !TD_WINDOWS +#include <arpa/inet.h> +#include <sys/socket.h> +#endif + +namespace td { +class SocketFd; +class IPAddress { + public: + IPAddress(); + + bool is_valid() const; + + const sockaddr *get_sockaddr() const; + size_t get_sockaddr_len() const; + int get_address_family() const; + Slice get_ip_str() const; + bool is_ipv4() const; + uint32 get_ipv4() const; + Slice get_ipv6() const; + int get_port() const; + void set_port(int port); + + IPAddress get_any_addr() const; + + Status init_ipv6_port(CSlice ipv6, int port) TD_WARN_UNUSED_RESULT; + Status init_ipv6_as_ipv4_port(CSlice ipv4, int port) TD_WARN_UNUSED_RESULT; + Status init_ipv4_port(CSlice ipv4, int port) TD_WARN_UNUSED_RESULT; + Status init_host_port(CSlice host, int port) TD_WARN_UNUSED_RESULT; + Status init_host_port(CSlice host, CSlice port) TD_WARN_UNUSED_RESULT; + Status init_host_port(CSlice host_port) TD_WARN_UNUSED_RESULT; + Status init_socket_address(const SocketFd &socket_fd) TD_WARN_UNUSED_RESULT; + Status init_peer_address(const SocketFd &socket_fd) TD_WARN_UNUSED_RESULT; + + friend bool operator==(const IPAddress &a, const IPAddress &b); + friend bool operator<(const IPAddress &a, const IPAddress &b); + + static CSlice ipv4_to_str(int32 ipv4); + + private: + union { + sockaddr_storage addr_; + sockaddr sockaddr_; + sockaddr_in ipv4_addr_; + sockaddr_in6 ipv6_addr_; + }; + bool is_valid_; + + Status init_sockaddr(sockaddr *addr, socklen_t len) TD_WARN_UNUSED_RESULT; + void init_ipv4_any(); + void init_ipv6_any(); +}; + +StringBuilder &operator<<(StringBuilder &builder, const IPAddress &address); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/Poll.h b/libs/tdlib/td/tdutils/td/utils/port/Poll.h new file mode 100644 index 0000000000..e23f4382d0 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Poll.h @@ -0,0 +1,35 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/port/detail/Epoll.h" +#include "td/utils/port/detail/KQueue.h" +#include "td/utils/port/detail/Poll.h" +#include "td/utils/port/detail/Select.h" +#include "td/utils/port/detail/WineventPoll.h" + +namespace td { + +// clang-format off + +#if TD_POLL_EPOLL + using Poll = detail::Epoll; +#elif TD_POLL_KQUEUE + using Poll = detail::KQueue; +#elif TD_POLL_WINEVENT + using Poll = detail::WineventPoll; +#elif TD_POLL_POLL + using Poll = detail::Poll; +#elif TD_POLL_SELECT + using Poll = detail::Select; +#endif + +// clang-format on + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/PollBase.h b/libs/tdlib/td/tdutils/td/utils/port/PollBase.h new file mode 100644 index 0000000000..eb71367ab9 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/PollBase.h @@ -0,0 +1,27 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/Fd.h" + +namespace td { +class PollBase { + public: + PollBase() = default; + PollBase(const PollBase &) = delete; + PollBase &operator=(const PollBase &) = delete; + PollBase(PollBase &&) = default; + PollBase &operator=(PollBase &&) = default; + virtual ~PollBase() = default; + virtual void init() = 0; + virtual void clear() = 0; + virtual void subscribe(const Fd &fd, Fd::Flags flags) = 0; + virtual void unsubscribe(const Fd &fd) = 0; + virtual void unsubscribe_before_close(const Fd &fd) = 0; + virtual void run(int timeout_ms) = 0; +}; +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/RwMutex.h b/libs/tdlib/td/tdutils/td/utils/port/RwMutex.h new file mode 100644 index 0000000000..eee5f3dcdb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/RwMutex.h @@ -0,0 +1,147 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +#if TD_PORT_POSIX +#include <pthread.h> +#endif + +namespace td { +class RwMutex { + public: + RwMutex() { + init(); + } + RwMutex(const RwMutex &) = delete; + RwMutex &operator=(const RwMutex &) = delete; + RwMutex(RwMutex &&other) { + init(); + other.clear(); + } + RwMutex &operator=(RwMutex &&other) { + other.clear(); + return *this; + } + ~RwMutex() { + clear(); + } + + bool empty() const { + return !is_valid_; + } + + void init(); + + void clear(); + + struct ReadUnlock { + void operator()(RwMutex *ptr) { + ptr->unlock_read_unsafe(); + } + }; + struct WriteUnlock { + void operator()(RwMutex *ptr) { + ptr->unlock_write_unsafe(); + } + }; + + using ReadLock = std::unique_ptr<RwMutex, ReadUnlock>; + using WriteLock = std::unique_ptr<RwMutex, WriteUnlock>; + + Result<ReadLock> lock_read() TD_WARN_UNUSED_RESULT { + lock_read_unsafe(); + return ReadLock(this); + } + + Result<WriteLock> lock_write() TD_WARN_UNUSED_RESULT { + lock_write_unsafe(); + return WriteLock(this); + } + + void lock_read_unsafe(); + + void lock_write_unsafe(); + + void unlock_read_unsafe(); + + void unlock_write_unsafe(); + + private: + bool is_valid_ = false; +#if TD_PORT_POSIX + pthread_rwlock_t mutex_; +#elif TD_PORT_WINDOWS + unique_ptr<SRWLOCK> mutex_; +#endif +}; + +inline void RwMutex::init() { + CHECK(empty()); + is_valid_ = true; +#if TD_PORT_POSIX + pthread_rwlock_init(&mutex_, nullptr); +#elif TD_PORT_WINDOWS + mutex_ = make_unique<SRWLOCK>(); + InitializeSRWLock(mutex_.get()); +#endif +} + +inline void RwMutex::clear() { + if (is_valid_) { +#if TD_PORT_POSIX + pthread_rwlock_destroy(&mutex_); +#elif TD_PORT_WINDOWS + mutex_.release(); +#endif + is_valid_ = false; + } +} + +inline void RwMutex::lock_read_unsafe() { + CHECK(!empty()); +// TODO error handling +#if TD_PORT_POSIX + pthread_rwlock_rdlock(&mutex_); +#elif TD_PORT_WINDOWS + AcquireSRWLockShared(mutex_.get()); +#endif +} + +inline void RwMutex::lock_write_unsafe() { + CHECK(!empty()); +#if TD_PORT_POSIX + pthread_rwlock_wrlock(&mutex_); +#elif TD_PORT_WINDOWS + AcquireSRWLockExclusive(mutex_.get()); +#endif +} + +inline void RwMutex::unlock_read_unsafe() { + CHECK(!empty()); +#if TD_PORT_POSIX + pthread_rwlock_unlock(&mutex_); +#elif TD_PORT_WINDOWS + ReleaseSRWLockShared(mutex_.get()); +#endif +} + +inline void RwMutex::unlock_write_unsafe() { + CHECK(!empty()); +#if TD_PORT_POSIX + pthread_rwlock_unlock(&mutex_); +#elif TD_PORT_WINDOWS + ReleaseSRWLockExclusive(mutex_.get()); +#endif +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/ServerSocketFd.cpp b/libs/tdlib/td/tdutils/td/utils/port/ServerSocketFd.cpp new file mode 100644 index 0000000000..ead43a3d4b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/ServerSocketFd.cpp @@ -0,0 +1,160 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/ServerSocketFd.h" + +#include "td/utils/port/config.h" + +#include "td/utils/logging.h" +#include "td/utils/port/IPAddress.h" + +#if TD_PORT_POSIX + +#include <arpa/inet.h> +#include <fcntl.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <unistd.h> + +#endif + +namespace td { + +Result<ServerSocketFd> ServerSocketFd::open(int32 port, CSlice addr) { + ServerSocketFd socket; + TRY_STATUS(socket.init(port, addr)); + return std::move(socket); +} + +const Fd &ServerSocketFd::get_fd() const { + return fd_; +} + +Fd &ServerSocketFd::get_fd() { + return fd_; +} + +int32 ServerSocketFd::get_flags() const { + return fd_.get_flags(); +} + +Status ServerSocketFd::get_pending_error() { + return fd_.get_pending_error(); +} + +Result<SocketFd> ServerSocketFd::accept() { +#if TD_PORT_POSIX + sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + int native_fd = fd_.get_native_fd(); + int r_fd = skip_eintr([&] { return ::accept(native_fd, reinterpret_cast<sockaddr *>(&addr), &addr_len); }); + auto accept_errno = errno; + if (r_fd >= 0) { + return SocketFd::from_native_fd(r_fd); + } + + if (accept_errno == EAGAIN +#if EAGAIN != EWOULDBLOCK + || accept_errno == EWOULDBLOCK +#endif + ) { + fd_.clear_flags(Fd::Read); + return Status::Error(-1, "Operation would block"); + } + + auto error = Status::PosixError(accept_errno, PSLICE() << "Accept from [fd = " << native_fd << "] has failed"); + switch (accept_errno) { + case EBADF: + case EFAULT: + case EINVAL: + case ENOTSOCK: + case EOPNOTSUPP: + LOG(FATAL) << error; + UNREACHABLE(); + break; + default: + LOG(ERROR) << error; + // fallthrough + case EMFILE: + case ENFILE: + case ECONNABORTED: //??? + fd_.clear_flags(Fd::Read); + fd_.update_flags(Fd::Close); + return std::move(error); + } +#elif TD_PORT_WINDOWS + TRY_RESULT(socket_fd, fd_.accept()); + return SocketFd(std::move(socket_fd)); +#endif +} + +void ServerSocketFd::close() { + fd_.close(); +} + +bool ServerSocketFd::empty() const { + return fd_.empty(); +} + +Status ServerSocketFd::init(int32 port, CSlice addr) { + IPAddress address; + TRY_STATUS(address.init_ipv4_port(addr, port)); + auto fd = socket(address.get_address_family(), SOCK_STREAM, 0); +#if TD_PORT_POSIX + if (fd == -1) { +#elif TD_PORT_WINDOWS + if (fd == INVALID_SOCKET) { +#endif + return OS_SOCKET_ERROR("Failed to create a socket"); + } + auto fd_quard = ScopeExit() + [fd]() { +#if TD_PORT_POSIX + ::close(fd); +#elif TD_PORT_WINDOWS + ::closesocket(fd); +#endif + }; + + TRY_STATUS(detail::set_native_socket_is_blocking(fd, false)); + + linger ling = {0, 0}; +#if TD_PORT_POSIX + int flags = 1; +#ifdef SO_REUSEPORT + setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, reinterpret_cast<const char *>(&flags), sizeof(flags)); +#endif +#elif TD_PORT_WINDOWS + BOOL flags = TRUE; +#endif + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char *>(&flags), sizeof(flags)); + setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast<const char *>(&flags), sizeof(flags)); + setsockopt(fd, SOL_SOCKET, SO_LINGER, reinterpret_cast<const char *>(&ling), sizeof(ling)); + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<const char *>(&flags), sizeof(flags)); + + int e_bind = bind(fd, address.get_sockaddr(), static_cast<socklen_t>(address.get_sockaddr_len())); + if (e_bind != 0) { + return OS_SOCKET_ERROR("Failed to bind a socket"); + } + + // TODO: magic constant + int e_listen = listen(fd, 8192); + if (e_listen != 0) { + return OS_SOCKET_ERROR("Failed to listen on a socket"); + } + +#if TD_PORT_POSIX + fd_ = Fd(fd, Fd::Mode::Owner); +#elif TD_PORT_WINDOWS + fd_ = Fd::create_server_socket_fd(fd, address.get_address_family()); +#endif + + fd_quard.dismiss(); + return Status::OK(); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/ServerSocketFd.h b/libs/tdlib/td/tdutils/td/utils/port/ServerSocketFd.h new file mode 100644 index 0000000000..67b43ad02d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/ServerSocketFd.h @@ -0,0 +1,43 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/Fd.h" +#include "td/utils/port/SocketFd.h" + +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +class ServerSocketFd { + public: + ServerSocketFd() = default; + ServerSocketFd(const ServerSocketFd &) = delete; + ServerSocketFd &operator=(const ServerSocketFd &) = delete; + ServerSocketFd(ServerSocketFd &&) = default; + ServerSocketFd &operator=(ServerSocketFd &&) = default; + + static Result<ServerSocketFd> open(int32 port, CSlice addr = CSlice("0.0.0.0")) TD_WARN_UNUSED_RESULT; + + const Fd &get_fd() const; + Fd &get_fd(); + int32 get_flags() const; + Status get_pending_error() TD_WARN_UNUSED_RESULT; + + Result<SocketFd> accept() TD_WARN_UNUSED_RESULT; + + void close(); + bool empty() const; + + private: + Fd fd_; + + Status init(int32 port, CSlice addr) TD_WARN_UNUSED_RESULT; +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/SocketFd.cpp b/libs/tdlib/td/tdutils/td/utils/port/SocketFd.cpp new file mode 100644 index 0000000000..790bcd1bbd --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/SocketFd.cpp @@ -0,0 +1,139 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/SocketFd.h" + +#include "td/utils/logging.h" + +#if TD_PORT_WINDOWS +#include "td/utils/misc.h" +#endif + +#if TD_PORT_POSIX +#include <arpa/inet.h> +#include <fcntl.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <unistd.h> +#endif + +namespace td { + +Result<SocketFd> SocketFd::open(const IPAddress &address) { + SocketFd socket; + TRY_STATUS(socket.init(address)); + return std::move(socket); +} + +#if TD_PORT_POSIX +Result<SocketFd> SocketFd::from_native_fd(int fd) { + auto fd_guard = ScopeExit() + [fd]() { ::close(fd); }; + + TRY_STATUS(detail::set_native_socket_is_blocking(fd, false)); + + // TODO remove copypaste + int flags = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char *>(&flags), sizeof(flags)); + setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast<const char *>(&flags), sizeof(flags)); + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<const char *>(&flags), sizeof(flags)); + // TODO: SO_REUSEADDR, SO_KEEPALIVE, TCP_NODELAY, SO_SNDBUF, SO_RCVBUF, TCP_QUICKACK, SO_LINGER + + fd_guard.dismiss(); + + SocketFd socket; + socket.fd_ = Fd(fd, Fd::Mode::Owner); + return std::move(socket); +} +#endif + +Status SocketFd::init(const IPAddress &address) { + auto fd = socket(address.get_address_family(), SOCK_STREAM, 0); +#if TD_PORT_POSIX + if (fd == -1) { +#elif TD_PORT_WINDOWS + if (fd == INVALID_SOCKET) { +#endif + return OS_SOCKET_ERROR("Failed to create a socket"); + } + auto fd_quard = ScopeExit() + [fd]() { +#if TD_PORT_POSIX + ::close(fd); +#elif TD_PORT_WINDOWS + ::closesocket(fd); +#endif + }; + + TRY_STATUS(detail::set_native_socket_is_blocking(fd, false)); + +#if TD_PORT_POSIX + int flags = 1; +#elif TD_PORT_WINDOWS + BOOL flags = TRUE; +#endif + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char *>(&flags), sizeof(flags)); + setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast<const char *>(&flags), sizeof(flags)); + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<const char *>(&flags), sizeof(flags)); + // TODO: SO_REUSEADDR, SO_KEEPALIVE, TCP_NODELAY, SO_SNDBUF, SO_RCVBUF, TCP_QUICKACK, SO_LINGER + +#if TD_PORT_POSIX + int e_connect = connect(fd, address.get_sockaddr(), static_cast<socklen_t>(address.get_sockaddr_len())); + if (e_connect == -1) { + auto connect_errno = errno; + if (connect_errno != EINPROGRESS) { + return Status::PosixError(connect_errno, PSLICE() << "Failed to connect to " << address); + } + } + fd_ = Fd(fd, Fd::Mode::Owner); +#elif TD_PORT_WINDOWS + auto bind_addr = address.get_any_addr(); + auto e_bind = bind(fd, bind_addr.get_sockaddr(), narrow_cast<int>(bind_addr.get_sockaddr_len())); + if (e_bind != 0) { + return OS_SOCKET_ERROR("Failed to bind a socket"); + } + + fd_ = Fd::create_socket_fd(fd); + fd_.connect(address); +#endif + + fd_quard.dismiss(); + return Status::OK(); +} + +const Fd &SocketFd::get_fd() const { + return fd_; +} + +Fd &SocketFd::get_fd() { + return fd_; +} + +void SocketFd::close() { + fd_.close(); +} + +bool SocketFd::empty() const { + return fd_.empty(); +} + +int32 SocketFd::get_flags() const { + return fd_.get_flags(); +} + +Status SocketFd::get_pending_error() { + return fd_.get_pending_error(); +} + +Result<size_t> SocketFd::write(Slice slice) { + return fd_.write(slice); +} + +Result<size_t> SocketFd::read(MutableSlice slice) { + return fd_.read(slice); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/SocketFd.h b/libs/tdlib/td/tdutils/td/utils/port/SocketFd.h new file mode 100644 index 0000000000..c88dd7d789 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/SocketFd.h @@ -0,0 +1,57 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/port/Fd.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +class SocketFd { + public: + SocketFd() = default; + SocketFd(const SocketFd &) = delete; + SocketFd &operator=(const SocketFd &) = delete; + SocketFd(SocketFd &&) = default; + SocketFd &operator=(SocketFd &&) = default; + + static Result<SocketFd> open(const IPAddress &address) TD_WARN_UNUSED_RESULT; + + const Fd &get_fd() const; + Fd &get_fd(); + + int32 get_flags() const; + + Status get_pending_error() TD_WARN_UNUSED_RESULT; + + Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT; + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT; + + void close(); + bool empty() const; + + private: + Fd fd_; + + friend class ServerSocketFd; + + Status init(const IPAddress &address) TD_WARN_UNUSED_RESULT; + +#if TD_PORT_POSIX + static Result<SocketFd> from_native_fd(int fd); +#endif +#if TD_PORT_WINDOWS + explicit SocketFd(Fd fd) : fd_(std::move(fd)) { + } +#endif +}; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/Stat.cpp b/libs/tdlib/td/tdutils/td/utils/port/Stat.cpp new file mode 100644 index 0000000000..edc882761b --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Stat.cpp @@ -0,0 +1,337 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/Stat.h" + +#include "td/utils/port/FileFd.h" + +#if TD_PORT_POSIX + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/Clocks.h" +#include "td/utils/ScopeGuard.h" + +#include <utility> + +#if TD_DARWIN +#include <mach/mach.h> +#include <sys/time.h> +#endif + +// We don't want warnings from system headers +#if TD_GCC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif +#include <sys/stat.h> +#if TD_GCC +#pragma GCC diagnostic pop +#endif + +#if TD_ANDROID || TD_TIZEN +#include <sys/syscall.h> +#endif + +namespace td { + +namespace detail { + +template <class...> +struct voider { + using type = void; +}; +template <class... T> +using void_t = typename voider<T...>::type; + +template <class T, class = void> +struct TimeNsec { + static std::pair<int, int> get(const T &) { + T().warning("Platform lacks support of precise access/modification file times, comment this line to continue"); + return {0, 0}; + } +}; + +template <class T> +struct TimeNsec<T, void_t<char, decltype(T::st_atimespec), decltype(T::st_mtimespec)>> { + static std::pair<decltype(decltype(T::st_atimespec)::tv_nsec), decltype(decltype(T::st_mtimespec)::tv_nsec)> get( + const T &s) { + return {s.st_atimespec.tv_nsec, s.st_mtimespec.tv_nsec}; + } +}; + +template <class T> +struct TimeNsec<T, void_t<short, decltype(T::st_atimensec), decltype(T::st_mtimensec)>> { + static std::pair<decltype(T::st_atimensec), decltype(T::st_mtimensec)> get(const T &s) { + return {s.st_atimensec, s.st_mtimensec}; + } +}; + +template <class T> +struct TimeNsec<T, void_t<int, decltype(T::st_atim), decltype(T::st_mtim)>> { + static std::pair<decltype(decltype(T::st_atim)::tv_nsec), decltype(decltype(T::st_mtim)::tv_nsec)> get(const T &s) { + return {s.st_atim.tv_nsec, s.st_mtim.tv_nsec}; + } +}; + +Stat from_native_stat(const struct ::stat &buf) { + auto time_nsec = TimeNsec<struct ::stat>::get(buf); + + Stat res; + res.atime_nsec_ = static_cast<uint64>(buf.st_atime) * 1000000000 + time_nsec.first; + res.mtime_nsec_ = static_cast<uint64>(buf.st_mtime) * 1000000000 + time_nsec.second / 1000 * 1000; + res.size_ = buf.st_size; + res.is_dir_ = (buf.st_mode & S_IFMT) == S_IFDIR; + res.is_reg_ = (buf.st_mode & S_IFMT) == S_IFREG; + return res; +} + +Stat fstat(int native_fd) { + struct ::stat buf; + int err = fstat(native_fd, &buf); + auto fstat_errno = errno; + LOG_IF(FATAL, err < 0) << Status::PosixError(fstat_errno, PSLICE() << "stat for fd " << native_fd << " failed"); + return detail::from_native_stat(buf); +} + +Status update_atime(int native_fd) { +#if TD_LINUX + timespec times[2]; + // access time + times[0].tv_nsec = UTIME_NOW; + // modify time + times[1].tv_nsec = UTIME_OMIT; + if (futimens(native_fd, times) < 0) { + auto status = OS_ERROR(PSLICE() << "futimens " << tag("fd", native_fd)); + LOG(WARNING) << status; + return status; + } + return Status::OK(); +#elif TD_DARWIN + auto info = fstat(native_fd); + timeval upd[2]; + auto now = Clocks::system(); + // access time + upd[0].tv_sec = static_cast<decltype(upd[0].tv_sec)>(now); + upd[0].tv_usec = static_cast<decltype(upd[0].tv_usec)>((now - static_cast<double>(upd[0].tv_sec)) * 1000000); + // modify time + upd[1].tv_sec = static_cast<decltype(upd[1].tv_sec)>(info.mtime_nsec_ / 1000000000ll); + upd[1].tv_usec = static_cast<decltype(upd[1].tv_usec)>(info.mtime_nsec_ % 1000000000ll / 1000); + if (futimes(native_fd, upd) < 0) { + auto status = OS_ERROR(PSLICE() << "futimes " << tag("fd", native_fd)); + LOG(WARNING) << status; + return status; + } + return Status::OK(); +#else + return Status::Error("Not supported"); +// timespec times[2]; +//// access time +// times[0].tv_nsec = UTIME_NOW; +//// modify time +// times[1].tv_nsec = UTIME_OMIT; +//// int err = syscall(__NR_utimensat, native_fd, nullptr, times, 0); +// if (futimens(native_fd, times) < 0) { +// auto status = OS_ERROR(PSLICE() << "futimens " << tag("fd", native_fd)); +// LOG(WARNING) << status; +// return status; +// } +// return Status::OK(); +#endif +} +} // namespace detail + +Status update_atime(CSlice path) { + TRY_RESULT(file, FileFd::open(path, FileFd::Flags::Read)); + SCOPE_EXIT { + file.close(); + }; + return detail::update_atime(file.get_native_fd()); +} + +Result<Stat> stat(CSlice path) { + struct ::stat buf; + if (stat(path.c_str(), &buf) < 0) { + return OS_ERROR(PSLICE() << "stat for " << tag("file", path) << " failed"); + } + return detail::from_native_stat(buf); +} + +Result<MemStat> mem_stat() { +#if TD_DARWIN + task_basic_info t_info; + mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + + if (KERN_SUCCESS != + task_info(mach_task_self(), TASK_BASIC_INFO, reinterpret_cast<task_info_t>(&t_info), &t_info_count)) { + return Status::Error("task_info failed"); + } + MemStat res; + res.resident_size_ = t_info.resident_size; + res.virtual_size_ = t_info.virtual_size; + res.resident_size_peak_ = 0; + res.virtual_size_peak_ = 0; + return res; +#elif TD_LINUX || TD_ANDROID || TD_TIZEN + TRY_RESULT(fd, FileFd::open("/proc/self/status", FileFd::Read)); + SCOPE_EXIT { + fd.close(); + }; + + constexpr int TMEM_SIZE = 10000; + char mem[TMEM_SIZE]; + TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1))); + CHECK(size < TMEM_SIZE - 1); + mem[size] = 0; + + const char *s = mem; + MemStat res; + while (*s) { + const char *name_begin = s; + while (*s != 0 && *s != '\n') { + s++; + } + auto name_end = name_begin; + while (is_alpha(*name_end)) { + name_end++; + } + Slice name(name_begin, name_end); + + uint64 *x = nullptr; + if (name == "VmPeak") { + x = &res.virtual_size_peak_; + } + if (name == "VmSize") { + x = &res.virtual_size_; + } + if (name == "VmHWM") { + x = &res.resident_size_peak_; + } + if (name == "VmRSS") { + x = &res.resident_size_; + } + if (x != nullptr) { + Slice value(name_end, s); + if (!value.empty() && value[0] == ':') { + value.remove_prefix(1); + } + value = trim(value); + value = split(value).first; + auto r_mem = to_integer_safe<uint64>(value); + if (r_mem.is_error()) { + LOG(ERROR) << "Failed to parse memory stats " << tag("name", name) << tag("value", value); + *x = static_cast<uint64>(-1); + } else { + *x = r_mem.ok() * 1024; // memory is in kB + } + } + if (*s == 0) { + break; + } + s++; + } + + return res; +#else + return Status::Error("Not supported"); +#endif +} + +#if TD_LINUX +Status cpu_stat_self(CpuStat &stat) { + TRY_RESULT(fd, FileFd::open("/proc/self/stat", FileFd::Read)); + SCOPE_EXIT { + fd.close(); + }; + + constexpr int TMEM_SIZE = 10000; + char mem[TMEM_SIZE]; + TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1))); + CHECK(size < TMEM_SIZE - 1); + mem[size] = 0; + + char *s = mem; + char *t = mem + size; + int pass_cnt = 0; + + while (pass_cnt < 15) { + if (pass_cnt == 13) { + stat.process_user_ticks = to_integer<uint64>(Slice(s, t)); + } + if (pass_cnt == 14) { + stat.process_system_ticks = to_integer<uint64>(Slice(s, t)); + } + while (*s && *s != ' ') { + s++; + } + if (*s == ' ') { + s++; + pass_cnt++; + } else { + return Status::Error("unexpected end of proc file"); + } + } + return Status::OK(); +} +Status cpu_stat_total(CpuStat &stat) { + TRY_RESULT(fd, FileFd::open("/proc/stat", FileFd::Read)); + SCOPE_EXIT { + fd.close(); + }; + + constexpr int TMEM_SIZE = 10000; + char mem[TMEM_SIZE]; + TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1))); + CHECK(size < TMEM_SIZE - 1); + mem[size] = 0; + + uint64 sum = 0, cur = 0; + for (size_t i = 0; i < size; i++) { + int c = mem[i]; + if (c >= '0' && c <= '9') { + cur = cur * 10 + (uint64)c - '0'; + } else { + sum += cur; + cur = 0; + if (c == '\n') { + break; + } + } + } + + stat.total_ticks = sum; + return Status::OK(); +} +#endif + +Result<CpuStat> cpu_stat() { +#if TD_LINUX + CpuStat stat; + TRY_STATUS(cpu_stat_self(stat)); + TRY_STATUS(cpu_stat_total(stat)); + return stat; +#else + return Status::Error("Not supported"); +#endif +} +} // namespace td +#endif + +#if TD_PORT_WINDOWS +namespace td { + +Result<Stat> stat(CSlice path) { + TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read)); + return fd.stat(); +} + +Result<CpuStat> cpu_stat() { + return Status::Error("Not supported"); +} + +} // namespace td +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/Stat.h b/libs/tdlib/td/tdutils/td/utils/port/Stat.h new file mode 100644 index 0000000000..d0a98db141 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/Stat.h @@ -0,0 +1,53 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +struct Stat { + bool is_dir_; + bool is_reg_; + int64 size_; + uint64 atime_nsec_; + uint64 mtime_nsec_; +}; + +Result<Stat> stat(CSlice path) TD_WARN_UNUSED_RESULT; + +struct CpuStat { + uint64 total_ticks{0}; + uint64 process_user_ticks{0}; + uint64 process_system_ticks{0}; +}; +Result<CpuStat> cpu_stat() TD_WARN_UNUSED_RESULT; + +#if TD_PORT_POSIX + +namespace detail { +Stat fstat(int native_fd); // TODO return Result<Stat> +} // namespace detail + +Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT; + +struct MemStat { + uint64 resident_size_ = 0; + uint64 resident_size_peak_ = 0; + uint64 virtual_size_ = 0; + uint64 virtual_size_peak_ = 0; +}; + +Result<MemStat> mem_stat() TD_WARN_UNUSED_RESULT; + +#endif + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/config.h b/libs/tdlib/td/tdutils/td/utils/port/config.h new file mode 100644 index 0000000000..0ffdb3c3bf --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/config.h @@ -0,0 +1,46 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/platform.h" + +// clang-format off + +#if TD_WINDOWS + #define TD_PORT_WINDOWS 1 +#else + #define TD_PORT_POSIX 1 +#endif + +#if TD_LINUX || TD_ANDROID || TD_TIZEN + #define TD_POLL_EPOLL 1 + #define TD_EVENTFD_LINUX 1 +#elif TD_CYGWIN + #define TD_POLL_SELECT 1 + #define TD_EVENTFD_BSD 1 +#elif TD_EMSCRIPTEN + #define TD_POLL_POLL 1 + #define TD_EVENTFD_UNSUPPORTED 1 +#elif TD_DARWIN + #define TD_POLL_KQUEUE 1 + #define TD_EVENTFD_BSD 1 +#elif TD_WINDOWS + #define TD_POLL_WINEVENT 1 + #define TD_EVENTFD_WINDOWS 1 +#else + #error "Poll's implementation is not defined" +#endif + +#if TD_EMSCRIPTEN + #define TD_THREAD_UNSUPPORTED 1 +#elif TD_TIZEN + #define TD_THREAD_PTHREAD 1 +#else + #define TD_THREAD_STL 1 +#endif + +// clang-format on diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/Epoll.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/Epoll.cpp new file mode 100644 index 0000000000..2ef026d164 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/Epoll.cpp @@ -0,0 +1,114 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/Epoll.h" + +char disable_linker_warning_about_empty_file_epoll_cpp TD_UNUSED; + +#ifdef TD_POLL_EPOLL + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +#include <unistd.h> + +namespace td { +namespace detail { +void Epoll::init() { + CHECK(epoll_fd == -1); + epoll_fd = epoll_create(1); + auto epoll_create_errno = errno; + LOG_IF(FATAL, epoll_fd == -1) << Status::PosixError(epoll_create_errno, "epoll_create failed"); + + events.resize(1000); +} + +void Epoll::clear() { + if (epoll_fd == -1) { + return; + } + events.clear(); + + close(epoll_fd); + epoll_fd = -1; +} + +void Epoll::subscribe(const Fd &fd, Fd::Flags flags) { + epoll_event event; + event.events = EPOLLHUP | EPOLLERR | EPOLLET; +#ifdef EPOLLRDHUP + event.events |= EPOLLRDHUP; +#endif + if (flags & Fd::Read) { + event.events |= EPOLLIN; + } + if (flags & Fd::Write) { + event.events |= EPOLLOUT; + } + auto native_fd = fd.get_native_fd(); + event.data.fd = native_fd; + int err = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, native_fd, &event); + auto epoll_ctl_errno = errno; + LOG_IF(FATAL, err == -1) << Status::PosixError(epoll_ctl_errno, "epoll_ctl ADD failed") << ", epoll_fd = " << epoll_fd + << ", fd = " << native_fd; +} + +void Epoll::unsubscribe(const Fd &fd) { + auto native_fd = fd.get_native_fd(); + int err = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, native_fd, nullptr); + auto epoll_ctl_errno = errno; + LOG_IF(FATAL, err == -1) << Status::PosixError(epoll_ctl_errno, "epoll_ctl DEL failed") << ", epoll_fd = " << epoll_fd + << ", fd = " << native_fd; +} + +void Epoll::unsubscribe_before_close(const Fd &fd) { + unsubscribe(fd); +} + +void Epoll::run(int timeout_ms) { + int ready_n = epoll_wait(epoll_fd, &events[0], static_cast<int>(events.size()), timeout_ms); + auto epoll_wait_errno = errno; + LOG_IF(FATAL, ready_n == -1 && epoll_wait_errno != EINTR) + << Status::PosixError(epoll_wait_errno, "epoll_wait failed"); + + for (int i = 0; i < ready_n; i++) { + Fd::Flags flags = 0; + epoll_event *event = &events[i]; + if (event->events & EPOLLIN) { + event->events &= ~EPOLLIN; + flags |= Fd::Read; + } + if (event->events & EPOLLOUT) { + event->events &= ~EPOLLOUT; + flags |= Fd::Write; + } +#ifdef EPOLLRDHUP + if (event->events & EPOLLRDHUP) { + event->events &= ~EPOLLRDHUP; + // flags |= Fd::Close; + // TODO + } +#endif + if (event->events & EPOLLHUP) { + event->events &= ~EPOLLHUP; + flags |= Fd::Close; + } + if (event->events & EPOLLERR) { + event->events &= ~EPOLLERR; + flags |= Fd::Error; + } + if (event->events) { + LOG(FATAL) << "Unsupported epoll events: " << event->events; + } + // LOG(DEBUG) << "Epoll event " << tag("fd", event->data.fd) << tag("flags", format::as_binary(flags)); + Fd(event->data.fd, Fd::Mode::Reference).update_flags_notify(flags); + } +} +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/Epoll.h b/libs/tdlib/td/tdutils/td/utils/port/detail/Epoll.h new file mode 100644 index 0000000000..db4f66e5a7 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/Epoll.h @@ -0,0 +1,51 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_POLL_EPOLL + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/PollBase.h" + +#include <sys/epoll.h> + +namespace td { +namespace detail { + +class Epoll final : public PollBase { + public: + Epoll() = default; + Epoll(const Epoll &) = delete; + Epoll &operator=(const Epoll &) = delete; + Epoll(Epoll &&) = delete; + Epoll &operator=(Epoll &&) = delete; + ~Epoll() override = default; + + void init() override; + + void clear() override; + + void subscribe(const Fd &fd, Fd::Flags flags) override; + + void unsubscribe(const Fd &fd) override; + + void unsubscribe_before_close(const Fd &fd) override; + + void run(int timeout_ms) override; + + private: + int epoll_fd = -1; + vector<struct epoll_event> events; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.cpp new file mode 100644 index 0000000000..d51e99ac0a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.cpp @@ -0,0 +1,93 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/EventFdBsd.h" + +char disable_linker_warning_about_empty_file_event_fd_bsd_cpp TD_UNUSED; + +#ifdef TD_EVENTFD_BSD + +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +#include <fcntl.h> +#include <sys/socket.h> +#include <sys/types.h> + +namespace td { +namespace detail { + +// TODO: it is extremely non optimal on Darwin. kqueue events should be used instead +void EventFdBsd::init() { + int fds[2]; + int err = socketpair(AF_UNIX, SOCK_STREAM, 0, fds); + auto socketpair_errno = errno; +#if TD_CYGWIN + // it looks like CYGWIN bug + int max_retries = 1000000; + while (err == -1 && socketpair_errno == EADDRINUSE && max_retries-- > 0) { + err = socketpair(AF_UNIX, SOCK_STREAM, 0, fds); + socketpair_errno = errno; + } +// LOG_IF(ERROR, max_retries < 1000000) << max_retries; +#endif + LOG_IF(FATAL, err == -1) << Status::PosixError(socketpair_errno, "socketpair failed"); + + detail::set_native_socket_is_blocking(fds[0], false).ensure(); + detail::set_native_socket_is_blocking(fds[1], false).ensure(); + + in_ = Fd(fds[0], Fd::Mode::Owner); + out_ = Fd(fds[1], Fd::Mode::Owner); +} + +bool EventFdBsd::empty() { + return in_.empty(); +} + +void EventFdBsd::close() { + in_.close(); + out_.close(); +} + +Status EventFdBsd::get_pending_error() { + return Status::OK(); +} + +const Fd &EventFdBsd::get_fd() const { + return out_; +} + +Fd &EventFdBsd::get_fd() { + return out_; +} + +void EventFdBsd::release() { + int value = 1; + auto result = in_.write(Slice(reinterpret_cast<const char *>(&value), sizeof(value))); + if (result.is_error()) { + LOG(FATAL) << "EventFdBsd write failed: " << result.error(); + } + size_t size = result.ok(); + if (size != sizeof(value)) { + LOG(FATAL) << "EventFdBsd write returned " << value << " instead of " << sizeof(value); + } +} + +void EventFdBsd::acquire() { + out_.update_flags(Fd::Read); + while (can_read(out_)) { + uint8 value[1024]; + auto result = out_.read(MutableSlice(value, sizeof(value))); + if (result.is_error()) { + LOG(FATAL) << "EventFdBsd read failed:" << result.error(); + } + } +} + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.h b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.h new file mode 100644 index 0000000000..08f7ddd308 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdBsd.h @@ -0,0 +1,47 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_EVENTFD_BSD + +#include "td/utils/common.h" +#include "td/utils/port/EventFdBase.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Status.h" + +namespace td { +namespace detail { + +class EventFdBsd final : public EventFdBase { + Fd in_; + Fd out_; + + public: + EventFdBsd() = default; + + void init() override; + + bool empty() override; + + void close() override; + + Status get_pending_error() override TD_WARN_UNUSED_RESULT; + + const Fd &get_fd() const override; + Fd &get_fd() override; + + void release() override; + + void acquire() override; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.cpp new file mode 100644 index 0000000000..fd08c9af08 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.cpp @@ -0,0 +1,74 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/EventFdLinux.h" + +char disable_linker_warning_about_empty_file_event_fd_linux_cpp TD_UNUSED; + +#ifdef TD_EVENTFD_LINUX + +#include "td/utils/logging.h" +#include "td/utils/Slice.h" + +#include <sys/eventfd.h> + +namespace td { +namespace detail { + +void EventFdLinux::init() { + int fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + auto eventfd_errno = errno; + LOG_IF(FATAL, fd == -1) << Status::PosixError(eventfd_errno, "eventfd call failed"); + + fd_ = Fd(fd, Fd::Mode::Owner); +} + +bool EventFdLinux::empty() { + return fd_.empty(); +} + +void EventFdLinux::close() { + fd_.close(); +} + +Status EventFdLinux::get_pending_error() { + return Status::OK(); +} + +const Fd &EventFdLinux::get_fd() const { + return fd_; +} + +Fd &EventFdLinux::get_fd() { + return fd_; +} + +void EventFdLinux::release() { + const uint64 value = 1; + // NB: write_unsafe is used, because release will be called from multiple threads + auto result = fd_.write_unsafe(Slice(reinterpret_cast<const char *>(&value), sizeof(value))); + if (result.is_error()) { + LOG(FATAL) << "EventFdLinux write failed: " << result.error(); + } + size_t size = result.ok(); + if (size != sizeof(value)) { + LOG(FATAL) << "EventFdLinux write returned " << value << " instead of " << sizeof(value); + } +} + +void EventFdLinux::acquire() { + uint64 res; + auto result = fd_.read(MutableSlice(reinterpret_cast<char *>(&res), sizeof(res))); + if (result.is_error()) { + LOG(FATAL) << "EventFdLinux read failed: " << result.error(); + } + fd_.clear_flags(Fd::Read); +} + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.h b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.h new file mode 100644 index 0000000000..3df7ce3a5d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdLinux.h @@ -0,0 +1,44 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_EVENTFD_LINUX + +#include "td/utils/common.h" +#include "td/utils/port/EventFdBase.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Status.h" + +namespace td { +namespace detail { + +class EventFdLinux final : public EventFdBase { + Fd fd_; + + public: + void init() override; + + bool empty() override; + + void close() override; + + Status get_pending_error() override TD_WARN_UNUSED_RESULT; + + const Fd &get_fd() const override; + Fd &get_fd() override; + + void release() override; + + void acquire() override; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.cpp new file mode 100644 index 0000000000..8adfd5a686 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.cpp @@ -0,0 +1,51 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/EventFdWindows.h" + +char disable_linker_warning_about_empty_file_event_fd_windows_cpp TD_UNUSED; + +#ifdef TD_EVENTFD_WINDOWS + +namespace td { +namespace detail { + +void EventFdWindows::init() { + fd_ = Fd::create_event_fd(); +} + +bool EventFdWindows::empty() { + return fd_.empty(); +} + +void EventFdWindows::close() { + fd_.close(); +} + +Status EventFdWindows::get_pending_error() { + return Status::OK(); +} + +const Fd &EventFdWindows::get_fd() const { + return fd_; +} + +Fd &EventFdWindows::get_fd() { + return fd_; +} + +void EventFdWindows::release() { + fd_.release(); +} + +void EventFdWindows::acquire() { + fd_.acquire(); +} + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.h b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.h new file mode 100644 index 0000000000..48e1c763b3 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/EventFdWindows.h @@ -0,0 +1,46 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_EVENTFD_WINDOWS + +#include "td/utils/common.h" +#include "td/utils/port/EventFdBase.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Status.h" + +namespace td { +namespace detail { + +class EventFdWindows final : public EventFdBase { + Fd fd_; + + public: + EventFdWindows() = default; + + void init() override; + + bool empty() override; + + void close() override; + + Status get_pending_error() override TD_WARN_UNUSED_RESULT; + + const Fd &get_fd() const override; + Fd &get_fd() override; + + void release() override; + + void acquire() override; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/KQueue.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/KQueue.cpp new file mode 100644 index 0000000000..351f8d7d6c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/KQueue.cpp @@ -0,0 +1,160 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/KQueue.h" + +char disable_linker_warning_about_empty_file_kqueue_cpp TD_UNUSED; + +#ifdef TD_POLL_KQUEUE + +#include "td/utils/logging.h" +#include "td/utils/Status.h" + +#include <utility> + +#include <unistd.h> + +namespace td { +namespace detail { + +KQueue::KQueue() { + kq = -1; +} +KQueue::~KQueue() { + clear(); +} +void KQueue::init() { + kq = kqueue(); + auto kqueue_errno = errno; + LOG_IF(FATAL, kq == -1) << Status::PosixError(kqueue_errno, "kqueue creation failed"); + + // TODO: const + events.resize(1000); + changes_n = 0; +} + +void KQueue::clear() { + if (kq == -1) { + return; + } + events.clear(); + close(kq); + kq = -1; +} + +int KQueue::update(int nevents, const timespec *timeout, bool may_fail) { + int err = kevent(kq, &events[0], changes_n, &events[0], nevents, timeout); + auto kevent_errno = errno; + + bool is_fatal_error = [&] { + if (err != -1) { + return false; + } + if (may_fail) { + return kevent_errno != ENOENT; + } + return kevent_errno != EINTR; + }(); + LOG_IF(FATAL, is_fatal_error) << Status::PosixError(kevent_errno, "kevent failed"); + + changes_n = 0; + if (err < 0) { + return 0; + } + return err; +} + +void KQueue::flush_changes(bool may_fail) { + if (!changes_n) { + return; + } + int n = update(0, nullptr, may_fail); + CHECK(n == 0); +} + +void KQueue::add_change(std::uintptr_t ident, int16 filter, uint16 flags, uint32 fflags, std::intptr_t data, + void *udata) { + if (changes_n == static_cast<int>(events.size())) { + flush_changes(); + } + EV_SET(&events[changes_n], ident, filter, flags, fflags, data, udata); + VLOG(fd) << "Subscribe [fd:" << ident << "] [filter:" << filter << "] [udata: " << udata << "]"; + changes_n++; +} + +void KQueue::subscribe(const Fd &fd, Fd::Flags flags) { + if (flags & Fd::Read) { + add_change(fd.get_native_fd(), EVFILT_READ, EV_ADD | EV_CLEAR, 0, 0, nullptr); + } + if (flags & Fd::Write) { + add_change(fd.get_native_fd(), EVFILT_WRITE, EV_ADD | EV_CLEAR, 0, 0, nullptr); + } +} + +void KQueue::invalidate(const Fd &fd) { + for (int i = 0; i < changes_n; i++) { + if (events[i].ident == static_cast<std::uintptr_t>(fd.get_native_fd())) { + changes_n--; + std::swap(events[i], events[changes_n]); + i--; + } + } +} + +void KQueue::unsubscribe(const Fd &fd) { + // invalidate(fd); + flush_changes(); + add_change(fd.get_native_fd(), EVFILT_READ, EV_DELETE, 0, 0, nullptr); + flush_changes(true); + add_change(fd.get_native_fd(), EVFILT_WRITE, EV_DELETE, 0, 0, nullptr); + flush_changes(true); +} + +void KQueue::unsubscribe_before_close(const Fd &fd) { + invalidate(fd); + + // just to avoid O(changes_n ^ 2) + if (changes_n != 0) { + flush_changes(); + } +} + +void KQueue::run(int timeout_ms) { + timespec timeout_data; + timespec *timeout_ptr; + if (timeout_ms == -1) { + timeout_ptr = nullptr; + } else { + timeout_data.tv_sec = timeout_ms / 1000; + timeout_data.tv_nsec = timeout_ms % 1000 * 1000000; + timeout_ptr = &timeout_data; + } + + int n = update(static_cast<int>(events.size()), timeout_ptr); + for (int i = 0; i < n; i++) { + struct kevent *event = &events[i]; + Fd::Flags flags = 0; + if (event->filter == EVFILT_WRITE) { + flags |= Fd::Write; + } + if (event->filter == EVFILT_READ) { + flags |= Fd::Read; + } + if (event->flags & EV_EOF) { + flags |= Fd::Close; + } + if (event->fflags & EV_ERROR) { + LOG(FATAL) << "EV_ERROR in kqueue is not supported"; + } + VLOG(fd) << "Event [fd:" << event->ident << "] [filter:" << event->filter << "] [udata: " << event->udata << "]"; + // LOG(WARNING) << "event->ident = " << event->ident << "event->filter = " << event->filter; + Fd(static_cast<int>(event->ident), Fd::Mode::Reference).update_flags_notify(flags); + } +} +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/KQueue.h b/libs/tdlib/td/tdutils/td/utils/port/detail/KQueue.h new file mode 100644 index 0000000000..e1b71e5fa5 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/KQueue.h @@ -0,0 +1,62 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_POLL_KQUEUE + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/PollBase.h" + +#include <cstdint> + +#include <sys/event.h> + +namespace td { +namespace detail { + +class KQueue final : public PollBase { + public: + KQueue(); + KQueue(const KQueue &) = delete; + KQueue &operator=(const KQueue &) = delete; + KQueue(KQueue &&) = delete; + KQueue &operator=(KQueue &&) = delete; + ~KQueue() override; + + void init() override; + + void clear() override; + + void subscribe(const Fd &fd, Fd::Flags flags) override; + + void unsubscribe(const Fd &fd) override; + + void unsubscribe_before_close(const Fd &fd) override; + + void run(int timeout_ms) override; + + private: + vector<struct kevent> events; + int changes_n; + int kq; + + int update(int nevents, const timespec *timeout, bool may_fail = false); + + void invalidate(const Fd &fd); + + void flush_changes(bool may_fail = false); + + void add_change(std::uintptr_t ident, int16 filter, uint16 flags, uint32 fflags, std::intptr_t data, void *udata); +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/Poll.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/Poll.cpp new file mode 100644 index 0000000000..87a7391802 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/Poll.cpp @@ -0,0 +1,92 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/Poll.h" + +char disable_linker_warning_about_empty_file_poll_cpp TD_UNUSED; + +#ifdef TD_POLL_POLL + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Status.h" + +namespace td { +namespace detail { + +void Poll::init() { +} + +void Poll::clear() { + pollfds_.clear(); +} + +void Poll::subscribe(const Fd &fd, Fd::Flags flags) { + unsubscribe(fd); + struct pollfd pollfd; + pollfd.fd = fd.get_native_fd(); + pollfd.events = 0; + if (flags & Fd::Read) { + pollfd.events |= POLLIN; + } + if (flags & Fd::Write) { + pollfd.events |= POLLOUT; + } + pollfd.revents = 0; + pollfds_.push_back(pollfd); +} + +void Poll::unsubscribe(const Fd &fd) { + for (auto it = pollfds_.begin(); it != pollfds_.end(); ++it) { + if (it->fd == fd.get_native_fd()) { + pollfds_.erase(it); + return; + } + } +} + +void Poll::unsubscribe_before_close(const Fd &fd) { + unsubscribe(fd); +} + +void Poll::run(int timeout_ms) { + int err = poll(pollfds_.data(), narrow_cast<int>(pollfds_.size()), timeout_ms); + auto poll_errno = errno; + LOG_IF(FATAL, err == -1 && poll_errno != EINTR) << Status::PosixError(poll_errno, "poll failed"); + + for (auto &pollfd : pollfds_) { + Fd::Flags flags = 0; + if (pollfd.revents & POLLIN) { + pollfd.revents &= ~POLLIN; + flags |= Fd::Read; + } + if (pollfd.revents & POLLOUT) { + pollfd.revents &= ~POLLOUT; + flags |= Fd::Write; + } + if (pollfd.revents & POLLHUP) { + pollfd.revents &= ~POLLHUP; + flags |= Fd::Close; + } + if (pollfd.revents & POLLERR) { + pollfd.revents &= ~POLLERR; + flags |= Fd::Error; + } + if (pollfd.revents & POLLNVAL) { + LOG(FATAL) << "Unexpected POLLNVAL " << tag("fd", pollfd.fd); + } + if (pollfd.revents) { + LOG(FATAL) << "Unsupported poll events: " << pollfd.revents; + } + Fd(pollfd.fd, Fd::Mode::Reference).update_flags_notify(flags); + } +} + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/Poll.h b/libs/tdlib/td/tdutils/td/utils/port/detail/Poll.h new file mode 100644 index 0000000000..32eca75399 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/Poll.h @@ -0,0 +1,50 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_POLL_POLL + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/PollBase.h" + +#include <poll.h> + +namespace td { +namespace detail { + +class Poll final : public PollBase { + public: + Poll() = default; + Poll(const Poll &) = delete; + Poll &operator=(const Poll &) = delete; + Poll(Poll &&) = delete; + Poll &operator=(Poll &&) = delete; + ~Poll() override = default; + + void init() override; + + void clear() override; + + void subscribe(const Fd &fd, Fd::Flags flags) override; + + void unsubscribe(const Fd &fd) override; + + void unsubscribe_before_close(const Fd &fd) override; + + void run(int timeout_ms) override; + + private: + vector<pollfd> pollfds_; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/Select.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/Select.cpp new file mode 100644 index 0000000000..b532a0464c --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/Select.cpp @@ -0,0 +1,119 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/Select.h" + +char disable_linker_warning_about_empty_file_select_cpp TD_UNUSED; + +#ifdef TD_POLL_SELECT + +#include "td/utils/logging.h" + +#include <utility> + +namespace td { +namespace detail { + +void Select::init() { + FD_ZERO(&all_fd_); + FD_ZERO(&read_fd_); + FD_ZERO(&write_fd_); + FD_ZERO(&except_fd_); + max_fd_ = -1; +} + +void Select::clear() { + fds_.clear(); +} + +void Select::subscribe(const Fd &fd, Fd::Flags flags) { + int native_fd = fd.get_native_fd(); + for (auto &it : fds_) { + CHECK(it.fd_ref.get_native_fd() != native_fd); + } + fds_.push_back(FdInfo{Fd(native_fd, Fd::Mode::Reference), flags}); + CHECK(0 <= native_fd && native_fd < FD_SETSIZE) << native_fd << " " << FD_SETSIZE; + FD_SET(native_fd, &all_fd_); + if (native_fd > max_fd_) { + max_fd_ = native_fd; + } +} + +void Select::unsubscribe(const Fd &fd) { + int native_fd = fd.get_native_fd(); + CHECK(0 <= native_fd && native_fd < FD_SETSIZE) << native_fd << " " << FD_SETSIZE; + FD_CLR(native_fd, &all_fd_); + FD_CLR(native_fd, &read_fd_); + FD_CLR(native_fd, &write_fd_); + FD_CLR(native_fd, &except_fd_); + while (max_fd_ >= 0 && !FD_ISSET(max_fd_, &all_fd_)) { + max_fd_--; + } + for (auto it = fds_.begin(); it != fds_.end();) { + if (it->fd_ref.get_native_fd() == native_fd) { + std::swap(*it, fds_.back()); + fds_.pop_back(); + break; + } else { + it++; + } + } +} + +void Select::unsubscribe_before_close(const Fd &fd) { + unsubscribe(fd); +} + +void Select::run(int timeout_ms) { + timeval timeout_data; + timeval *timeout_ptr; + if (timeout_ms == -1) { + timeout_ptr = nullptr; + } else { + timeout_data.tv_sec = timeout_ms / 1000; + timeout_data.tv_usec = timeout_ms % 1000 * 1000; + timeout_ptr = &timeout_data; + } + + for (auto &it : fds_) { + int native_fd = it.fd_ref.get_native_fd(); + Fd::Flags fd_flags = it.fd_ref.get_flags(); + if ((it.flags & Fd::Write) && !(fd_flags & Fd::Write)) { + FD_SET(native_fd, &write_fd_); + } else { + FD_CLR(native_fd, &write_fd_); + } + if ((it.flags & Fd::Read) && !(fd_flags & Fd::Read)) { + FD_SET(native_fd, &read_fd_); + } else { + FD_CLR(native_fd, &read_fd_); + } + FD_SET(native_fd, &except_fd_); + } + + select(max_fd_ + 1, &read_fd_, &write_fd_, &except_fd_, timeout_ptr); + for (auto &it : fds_) { + int native_fd = it.fd_ref.get_native_fd(); + Fd::Flags flags = 0; + if (FD_ISSET(native_fd, &read_fd_)) { + flags |= Fd::Read; + } + if (FD_ISSET(native_fd, &write_fd_)) { + flags |= Fd::Write; + } + if (FD_ISSET(native_fd, &except_fd_)) { + flags |= Fd::Error; + } + if (flags != 0) { + it.fd_ref.update_flags_notify(flags); + } + } +} + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/Select.h b/libs/tdlib/td/tdutils/td/utils/port/detail/Select.h new file mode 100644 index 0000000000..17f2876f3a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/Select.h @@ -0,0 +1,59 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_POLL_SELECT + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/PollBase.h" + +#include <sys/select.h> + +namespace td { +namespace detail { + +class Select final : public PollBase { + public: + Select() = default; + Select(const Select &) = delete; + Select &operator=(const Select &) = delete; + Select(Select &&) = delete; + Select &operator=(Select &&) = delete; + ~Select() override = default; + + void init() override; + + void clear() override; + + void subscribe(const Fd &fd, Fd::Flags flags) override; + + void unsubscribe(const Fd &fd) override; + + void unsubscribe_before_close(const Fd &fd) override; + + void run(int timeout_ms) override; + + private: + struct FdInfo { + Fd fd_ref; + Fd::Flags flags; + }; + vector<FdInfo> fds_; + fd_set all_fd_; + fd_set read_fd_; + fd_set write_fd_; + fd_set except_fd_; + int max_fd_; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.cpp new file mode 100644 index 0000000000..d949945e1d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.cpp @@ -0,0 +1,52 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/ThreadIdGuard.h" + +#include "td/utils/logging.h" +#include "td/utils/port/thread_local.h" + +#include <array> +#include <mutex> + +namespace td { +namespace detail { +class ThreadIdManager { + public: + int32 register_thread() { + std::lock_guard<std::mutex> guard(mutex_); + for (size_t i = 0; i < is_id_used_.size(); i++) { + if (!is_id_used_[i]) { + is_id_used_[i] = true; + return static_cast<int32>(i + 1); + } + } + LOG(FATAL) << "Cannot create more than " << max_thread_count() << " threads"; + return 0; + } + void unregister_thread(int32 thread_id) { + thread_id--; + std::lock_guard<std::mutex> guard(mutex_); + CHECK(is_id_used_.at(thread_id)); + is_id_used_[thread_id] = false; + } + + private: + std::mutex mutex_; + std::array<bool, max_thread_count()> is_id_used_{{false}}; +}; +static ThreadIdManager thread_id_manager; + +ThreadIdGuard::ThreadIdGuard() { + thread_id_ = thread_id_manager.register_thread(); + set_thread_id(thread_id_); +} +ThreadIdGuard::~ThreadIdGuard() { + thread_id_manager.unregister_thread(thread_id_); + set_thread_id(0); +} +} // namespace detail +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.h b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.h new file mode 100644 index 0000000000..434bd5ac4d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadIdGuard.h @@ -0,0 +1,26 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +namespace td { +namespace detail { +class ThreadIdGuard { + public: + ThreadIdGuard(); + ~ThreadIdGuard(); + ThreadIdGuard(const ThreadIdGuard &) = delete; + ThreadIdGuard &operator=(const ThreadIdGuard &) = delete; + ThreadIdGuard(ThreadIdGuard &&) = delete; + ThreadIdGuard &operator=(ThreadIdGuard &&) = delete; + + private: + int32 thread_id_; +}; +} // namespace detail +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadPthread.h b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadPthread.h new file mode 100644 index 0000000000..e42efc3771 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadPthread.h @@ -0,0 +1,90 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once +#include "td/utils/port/config.h" + +#ifdef TD_THREAD_PTHREAD + +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/MovableValue.h" +#include "td/utils/port/detail/ThreadIdGuard.h" +#include "td/utils/port/thread_local.h" + +#include <tuple> +#include <type_traits> +#include <utility> + +#include <pthread.h> +#include <sched.h> + +namespace td { +namespace detail { +class ThreadPthread { + public: + ThreadPthread() = default; + ThreadPthread(const ThreadPthread &other) = delete; + ThreadPthread &operator=(const ThreadPthread &other) = delete; + ThreadPthread(ThreadPthread &&) = default; + ThreadPthread &operator=(ThreadPthread &&) = default; + template <class Function, class... Args> + explicit ThreadPthread(Function &&f, Args &&... args) { + func_ = std::make_unique<std::unique_ptr<Destructor>>( + create_destructor([args = std::make_tuple(decay_copy(std::forward<Function>(f)), + decay_copy(std::forward<Args>(args))...)]() mutable { + invoke_tuple(std::move(args)); + clear_thread_locals(); + })); + pthread_create(&thread_, nullptr, run_thread, func_.get()); + is_inited_ = true; + } + void join() { + if (is_inited_.get()) { + is_inited_ = false; + pthread_join(thread_, nullptr); + } + } + ~ThreadPthread() { + join(); + } + + static unsigned hardware_concurrency() { + return 8; + } + + using id = pthread_t; + + private: + MovableValue<bool> is_inited_; + pthread_t thread_; + std::unique_ptr<std::unique_ptr<Destructor>> func_; + + template <class T> + std::decay_t<T> decay_copy(T &&v) { + return std::forward<T>(v); + } + + static void *run_thread(void *ptr) { + ThreadIdGuard thread_id_guard; + auto func = static_cast<decltype(func_.get())>(ptr); + func->reset(); + return nullptr; + } +}; + +namespace this_thread_pthread { +inline void yield() { + sched_yield(); +} +inline ThreadPthread::id get_id() { + return pthread_self(); +} +} // namespace this_thread_pthread +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadStl.h b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadStl.h new file mode 100644 index 0000000000..64bf3213cf --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/ThreadStl.h @@ -0,0 +1,64 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once +#include "td/utils/port/config.h" + +#ifdef TD_THREAD_STL + +#include "td/utils/common.h" +#include "td/utils/invoke.h" +#include "td/utils/port/detail/ThreadIdGuard.h" +#include "td/utils/port/thread_local.h" + +#include <thread> +#include <tuple> +#include <type_traits> +#include <utility> + +namespace td { +namespace detail { +class ThreadStl { + public: + ThreadStl() = default; + ThreadStl(const ThreadStl &other) = delete; + ThreadStl &operator=(const ThreadStl &other) = delete; + ThreadStl(ThreadStl &&) = default; + ThreadStl &operator=(ThreadStl &&) = default; + ~ThreadStl() = default; + template <class Function, class... Args> + explicit ThreadStl(Function &&f, Args &&... args) { + thread_ = std::thread([args = std::make_tuple(decay_copy(std::forward<Function>(f)), + decay_copy(std::forward<Args>(args))...)]() mutable { + ThreadIdGuard thread_id_guard; + invoke_tuple(std::move(args)); + clear_thread_locals(); + }); + } + + void join() { + thread_.join(); + } + + static unsigned hardware_concurrency() { + return std::thread::hardware_concurrency(); + } + + using id = std::thread::id; + + private: + std::thread thread_; + + template <class T> + std::decay_t<T> decay_copy(T &&v) { + return std::forward<T>(v); + } +}; +namespace this_thread_stl = std::this_thread; +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.cpp b/libs/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.cpp new file mode 100644 index 0000000000..8f443d29ab --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.cpp @@ -0,0 +1,97 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/detail/WineventPoll.h" + +char disable_linker_warning_about_empty_file_wineventpoll_cpp TD_UNUSED; + +#ifdef TD_POLL_WINEVENT + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/PollBase.h" +#include "td/utils/port/sleep.h" +#include "td/utils/Status.h" + +#include <utility> + +namespace td { +namespace detail { + +void WineventPoll::init() { + clear(); +} + +void WineventPoll::clear() { + fds_.clear(); +} + +void WineventPoll::subscribe(const Fd &fd, Fd::Flags flags) { + for (auto &it : fds_) { + if (it.fd_ref.get_key() == fd.get_key()) { + it.flags = flags; + return; + } + } + fds_.push_back({fd.clone(), flags}); +} + +void WineventPoll::unsubscribe(const Fd &fd) { + for (auto it = fds_.begin(); it != fds_.end(); ++it) { + if (it->fd_ref.get_key() == fd.get_key()) { + std::swap(*it, fds_.back()); + fds_.pop_back(); + return; + } + } +} + +void WineventPoll::unsubscribe_before_close(const Fd &fd) { + unsubscribe(fd); +} + +void WineventPoll::run(int timeout_ms) { + vector<std::pair<size_t, Fd::Flag>> events_desc; + vector<HANDLE> events; + for (size_t i = 0; i < fds_.size(); i++) { + auto &fd_info = fds_[i]; + if (fd_info.flags & Fd::Flag::Write) { + events_desc.emplace_back(i, Fd::Flag::Write); + events.push_back(fd_info.fd_ref.get_write_event()); + } + if (fd_info.flags & Fd::Flag::Read) { + events_desc.emplace_back(i, Fd::Flag::Read); + events.push_back(fd_info.fd_ref.get_read_event()); + } + } + if (events.empty()) { + usleep_for(timeout_ms * 1000); + return; + } + + auto status = WaitForMultipleObjects(narrow_cast<DWORD>(events.size()), events.data(), false, timeout_ms); + if (status == WAIT_FAILED) { + auto error = OS_ERROR("WaitForMultipleObjects failed"); + LOG(FATAL) << events.size() << " " << timeout_ms << " " << error; + } + for (size_t i = 0; i < events.size(); i++) { + if (WaitForSingleObject(events[i], 0) == WAIT_OBJECT_0) { + auto &fd = fds_[events_desc[i].first].fd_ref; + if (events_desc[i].second == Fd::Flag::Read) { + fd.on_read_event(); + } else { + fd.on_write_event(); + } + } + } +} + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.h b/libs/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.h new file mode 100644 index 0000000000..ecc93f33fa --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/detail/WineventPoll.h @@ -0,0 +1,52 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#ifdef TD_POLL_WINEVENT + +#include "td/utils/common.h" +#include "td/utils/port/Fd.h" +#include "td/utils/port/PollBase.h" + +namespace td { +namespace detail { + +class WineventPoll final : public PollBase { + public: + WineventPoll() = default; + WineventPoll(const WineventPoll &) = delete; + WineventPoll &operator=(const WineventPoll &) = delete; + WineventPoll(WineventPoll &&) = delete; + WineventPoll &operator=(WineventPoll &&) = delete; + ~WineventPoll() override = default; + + void init() override; + + void clear() override; + + void subscribe(const Fd &fd, Fd::Flags flags) override; + + void unsubscribe(const Fd &fd) override; + + void unsubscribe_before_close(const Fd &fd) override; + + void run(int timeout_ms) override; + + private: + struct FdInfo { + Fd fd_ref; + Fd::Flags flags; + }; + vector<FdInfo> fds_; +}; + +} // namespace detail +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/path.cpp b/libs/tdlib/td/tdutils/td/utils/port/path.cpp new file mode 100644 index 0000000000..8b169fefa4 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/path.cpp @@ -0,0 +1,383 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/path.h" + +#include "td/utils/port/Fd.h" + +#if TD_WINDOWS +#include "td/utils/Random.h" +#endif + +#if TD_PORT_POSIX + +#include <limits.h> +#include <stdio.h> + +// We don't want warnings from system headers +#if TD_GCC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif +#include <sys/stat.h> +#if TD_GCC +#pragma GCC diagnostic pop +#endif + +#include <sys/types.h> +#include <unistd.h> + +#endif + +#include <cstdlib> + +namespace td { + +static string temporary_dir; + +Status set_temporary_dir(CSlice dir) { + string input_dir = dir.str(); + if (!dir.empty() && dir.back() != TD_DIR_SLASH) { + input_dir += TD_DIR_SLASH; + } + TRY_STATUS(mkpath(input_dir, 0750)); + TRY_RESULT(real_dir, realpath(input_dir)); + temporary_dir = std::move(real_dir); + return Status::OK(); +} + +Status mkpath(CSlice path, int32 mode) { + Status first_error = Status::OK(); + Status last_error = Status::OK(); + for (size_t i = 1; i < path.size(); i++) { + if (path[i] == TD_DIR_SLASH) { + last_error = mkdir(path.substr(0, i).str(), mode); + if (last_error.is_error() && first_error.is_ok()) { + first_error = last_error.clone(); + } + } + } + if (last_error.is_error()) { + return first_error; + } + return Status::OK(); +} + +#if TD_PORT_POSIX + +Status mkdir(CSlice dir, int32 mode) { + int mkdir_res = skip_eintr([&] { return ::mkdir(dir.c_str(), static_cast<mode_t>(mode)); }); + if (mkdir_res == 0) { + return Status::OK(); + } + auto mkdir_errno = errno; + if (mkdir_errno == EEXIST) { + // TODO check that it is a directory + return Status::OK(); + } + return Status::PosixError(mkdir_errno, PSLICE() << "Can't create directory \"" << dir << '"'); +} + +Status rename(CSlice from, CSlice to) { + int rename_res = skip_eintr([&] { return ::rename(from.c_str(), to.c_str()); }); + if (rename_res < 0) { + return OS_ERROR(PSLICE() << "Can't rename \"" << from << "\" to \"" << to << '\"'); + } + return Status::OK(); +} + +Result<string> realpath(CSlice slice, bool ignore_access_denied) { + char full_path[PATH_MAX + 1]; + string res; + char *err = skip_eintr_cstr([&] { return ::realpath(slice.c_str(), full_path); }); + if (err != full_path) { + if (ignore_access_denied && errno == EACCES) { + res = slice.str(); + } else { + return OS_ERROR(PSLICE() << "Realpath failed for \"" << slice << '"'); + } + } else { + res = full_path; + } + if (res.empty()) { + return Status::Error("Empty path"); + } + if (!slice.empty() && slice.end()[-1] == TD_DIR_SLASH) { + if (res.back() != TD_DIR_SLASH) { + res += TD_DIR_SLASH; + } + } + return res; +} + +Status chdir(CSlice dir) { + int chdir_res = skip_eintr([&] { return ::chdir(dir.c_str()); }); + if (chdir_res) { + return OS_ERROR(PSLICE() << "Can't change directory to \"" << dir << '"'); + } + return Status::OK(); +} + +Status rmdir(CSlice dir) { + int rmdir_res = skip_eintr([&] { return ::rmdir(dir.c_str()); }); + if (rmdir_res) { + return OS_ERROR(PSLICE() << "Can't delete directory \"" << dir << '"'); + } + return Status::OK(); +} + +Status unlink(CSlice path) { + int unlink_res = skip_eintr([&] { return ::unlink(path.c_str()); }); + if (unlink_res) { + return OS_ERROR(PSLICE() << "Can't unlink \"" << path << '"'); + } + return Status::OK(); +} + +CSlice get_temporary_dir() { + static bool is_inited = [] { + if (temporary_dir.empty()) { + const char *s = std::getenv("TMPDIR"); + if (s != nullptr && s[0] != '\0') { + temporary_dir = s; + } else if (P_tmpdir != nullptr && P_tmpdir[0] != '\0') { + temporary_dir = P_tmpdir; + } else { + return false; + } + } + if (temporary_dir.size() > 1 && temporary_dir.back() == TD_DIR_SLASH) { + temporary_dir.pop_back(); + } + return true; + }(); + LOG_IF(FATAL, !is_inited) << "Can't find temporary directory"; + return temporary_dir; +} + +Result<std::pair<FileFd, string>> mkstemp(CSlice dir) { + if (dir.empty()) { + dir = get_temporary_dir(); + if (dir.empty()) { + return Status::Error("Can't find temporary directory"); + } + } + + TRY_RESULT(dir_real, realpath(dir)); + CHECK(!dir_real.empty()); + + string file_pattern; + file_pattern.reserve(dir_real.size() + 14); + file_pattern = dir_real; + if (file_pattern.back() != TD_DIR_SLASH) { + file_pattern += TD_DIR_SLASH; + } + file_pattern += "tmpXXXXXXXXXX"; + + int fd = skip_eintr([&] { return ::mkstemp(&file_pattern[0]); }); + if (fd == -1) { + return OS_ERROR(PSLICE() << "Can't create temporary file \"" << file_pattern << '"'); + } + if (close(fd)) { + return OS_ERROR(PSLICE() << "Can't close temporary file \"" << file_pattern << '"'); + } + // TODO create file from fd + TRY_RESULT(file, FileFd::open(file_pattern, FileFd::Write | FileFd::Truncate | FileFd::Append)); + return std::make_pair(std::move(file), std::move(file_pattern)); +} + +Result<string> mkdtemp(CSlice dir, Slice prefix) { + if (dir.empty()) { + dir = get_temporary_dir(); + if (dir.empty()) { + return Status::Error("Can't find temporary directory"); + } + } + + TRY_RESULT(dir_real, realpath(dir)); + CHECK(!dir_real.empty()); + + string dir_pattern; + dir_pattern.reserve(dir_real.size() + prefix.size() + 7); + dir_pattern = dir_real; + if (dir_pattern.back() != TD_DIR_SLASH) { + dir_pattern += TD_DIR_SLASH; + } + dir_pattern.append(prefix.begin(), prefix.size()); + dir_pattern += "XXXXXX"; + + char *result = skip_eintr_cstr([&] { return ::mkdtemp(&dir_pattern[0]); }); + if (result == nullptr) { + return OS_ERROR(PSLICE() << "Can't create temporary directory \"" << dir_pattern << '"'); + } + return result; +} + +#endif + +#if TD_PORT_WINDOWS + +Status mkdir(CSlice dir, int32 mode) { + TRY_RESULT(wdir, to_wstring(dir)); + auto status = CreateDirectoryW(wdir.c_str(), nullptr); + if (status == 0 && GetLastError() != ERROR_ALREADY_EXISTS) { + return OS_ERROR(PSLICE() << "Can't create directory \"" << dir << '"'); + } + return Status::OK(); +} + +Status rename(CSlice from, CSlice to) { + TRY_RESULT(wfrom, to_wstring(from)); + TRY_RESULT(wto, to_wstring(to)); + auto status = MoveFileExW(wfrom.c_str(), wto.c_str(), MOVEFILE_REPLACE_EXISTING); + if (status == 0) { + return OS_ERROR(PSLICE() << "Can't rename \"" << from << "\" to \"" << to << '\"'); + } + return Status::OK(); +} + +Result<string> realpath(CSlice slice, bool ignore_access_denied) { + wchar_t buf[MAX_PATH + 1]; + TRY_RESULT(wslice, to_wstring(slice)); + auto status = GetFullPathNameW(wslice.c_str(), MAX_PATH, buf, nullptr); + string res; + if (status == 0) { + if (ignore_access_denied && errno == ERROR_ACCESS_DENIED) { + res = slice.str(); + } else { + return OS_ERROR(PSLICE() << "GetFullPathNameW failed for \"" << slice << '"'); + } + } else { + TRY_RESULT(t_res, from_wstring(buf)); + res = std::move(t_res); + } + if (res.empty()) { + return Status::Error("Empty path"); + } + if (!slice.empty() && slice.end()[-1] == TD_DIR_SLASH) { + if (res.back() != TD_DIR_SLASH) { + res += TD_DIR_SLASH; + } + } + return res; +} + +Status chdir(CSlice dir) { + TRY_RESULT(wdir, to_wstring(dir)); + auto res = SetCurrentDirectoryW(wdir.c_str()); + if (res == 0) { + return OS_ERROR(PSLICE() << "Can't change directory to \"" << dir << '"'); + } + return Status::OK(); +} + +Status rmdir(CSlice dir) { + TRY_RESULT(wdir, to_wstring(dir)); + int status = RemoveDirectoryW(wdir.c_str()); + if (!status) { + return OS_ERROR(PSLICE() << "Can't delete directory \"" << dir << '"'); + } + return Status::OK(); +} + +Status unlink(CSlice path) { + TRY_RESULT(wpath, to_wstring(path)); + int status = DeleteFileW(wpath.c_str()); + if (!status) { + return OS_ERROR(PSLICE() << "Can't unlink \"" << path << '"'); + } + return Status::OK(); +} + +CSlice get_temporary_dir() { + static bool is_inited = [] { + if (temporary_dir.empty()) { + wchar_t buf[MAX_PATH + 1]; + if (GetTempPathW(MAX_PATH, buf) == 0) { + auto error = OS_ERROR("GetTempPathW failed"); + LOG(FATAL) << error; + } + auto rs = from_wstring(buf); + LOG_IF(FATAL, rs.is_error()) << "GetTempPathW failed: " << rs.error(); + temporary_dir = rs.ok(); + } + if (temporary_dir.size() > 1 && temporary_dir.back() == TD_DIR_SLASH) { + temporary_dir.pop_back(); + } + return true; + }(); + LOG_IF(FATAL, !is_inited) << "Can't find temporary directory"; + return temporary_dir; +} + +Result<string> mkdtemp(CSlice dir, Slice prefix) { + if (dir.empty()) { + dir = get_temporary_dir(); + if (dir.empty()) { + return Status::Error("Can't find temporary directory"); + } + } + + TRY_RESULT(dir_real, realpath(dir)); + CHECK(!dir_real.empty()); + + string dir_pattern; + dir_pattern.reserve(dir_real.size() + prefix.size() + 7); + dir_pattern = dir_real; + if (dir_pattern.back() != TD_DIR_SLASH) { + dir_pattern += TD_DIR_SLASH; + } + dir_pattern.append(prefix.begin(), prefix.size()); + + for (auto it = 0; it < 20; it++) { + auto path = dir_pattern; + for (int i = 0; i < 6 + it / 5; i++) { + path += static_cast<char>(Random::fast('a', 'z')); + } + auto status = mkdir(path); + if (status.is_ok()) { + return path; + } + } + return Status::Error(PSLICE() << "Can't create temporary directory \"" << dir_pattern << '"'); +} + +Result<std::pair<FileFd, string>> mkstemp(CSlice dir) { + if (dir.empty()) { + dir = get_temporary_dir(); + if (dir.empty()) { + return Status::Error("Can't find temporary directory"); + } + } + + TRY_RESULT(dir_real, realpath(dir)); + CHECK(!dir_real.empty()); + + string file_pattern; + file_pattern.reserve(dir_real.size() + 14); + file_pattern = dir_real; + if (file_pattern.back() != TD_DIR_SLASH) { + file_pattern += TD_DIR_SLASH; + } + file_pattern += "tmp"; + + for (auto it = 0; it < 20; it++) { + auto path = file_pattern; + for (int i = 0; i < 6 + it / 5; i++) { + path += static_cast<char>(Random::fast('a', 'z')); + } + auto r_file = FileFd::open(path, FileFd::Write | FileFd::Read | FileFd::CreateNew); + if (r_file.is_ok()) { + return std::make_pair(r_file.move_as_ok(), path); + } + } + + return Status::Error(PSLICE() << "Can't create temporary file \"" << file_pattern << '"'); +} + +#endif + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/path.h b/libs/tdlib/td/tdutils/td/utils/port/path.h new file mode 100644 index 0000000000..47b7d3a350 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/path.h @@ -0,0 +1,225 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/ScopeGuard.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <utility> + +#if TD_PORT_POSIX +#include <dirent.h> +#include <sys/types.h> +#endif + +#if TD_DARWIN +#include <sys/syslimits.h> +#endif + +#if TD_PORT_WINDOWS +#include "td/utils/port/wstring_convert.h" + +#include <string> +#endif + +namespace td { + +Status mkdir(CSlice dir, int32 mode = 0700) TD_WARN_UNUSED_RESULT; +Status mkpath(CSlice path, int32 mode = 0700) TD_WARN_UNUSED_RESULT; +Status rename(CSlice from, CSlice to) TD_WARN_UNUSED_RESULT; +Result<string> realpath(CSlice slice, bool ignore_access_denied = false) TD_WARN_UNUSED_RESULT; +Status chdir(CSlice dir) TD_WARN_UNUSED_RESULT; +Status rmdir(CSlice dir) TD_WARN_UNUSED_RESULT; +Status unlink(CSlice path) TD_WARN_UNUSED_RESULT; +Status set_temporary_dir(CSlice dir) TD_WARN_UNUSED_RESULT; +CSlice get_temporary_dir(); +Result<std::pair<FileFd, string>> mkstemp(CSlice dir) TD_WARN_UNUSED_RESULT; +Result<string> mkdtemp(CSlice dir, Slice prefix) TD_WARN_UNUSED_RESULT; + +template <class Func> +Status walk_path(CSlice path, Func &func) TD_WARN_UNUSED_RESULT; + +#if TD_PORT_POSIX + +// TODO move details somewhere else +namespace detail { +template <class Func> +Status walk_path_dir(string &path, FileFd fd, Func &&func) TD_WARN_UNUSED_RESULT; +template <class Func> +Status walk_path_dir(string &path, Func &&func) TD_WARN_UNUSED_RESULT; +template <class Func> +Status walk_path_file(string &path, Func &&func) TD_WARN_UNUSED_RESULT; +template <class Func> +Status walk_path(string &path, Func &&func) TD_WARN_UNUSED_RESULT; + +template <class Func> +Status walk_path_subdir(string &path, DIR *dir, Func &&func) { + while (true) { + errno = 0; + auto *entry = readdir(dir); + auto readdir_errno = errno; + if (readdir_errno) { + return Status::PosixError(readdir_errno, "readdir"); + } + if (entry == nullptr) { + return Status::OK(); + } + Slice name = Slice(&*entry->d_name); + if (name == "." || name == "..") { + continue; + } + auto size = path.size(); + if (path.back() != TD_DIR_SLASH) { + path += TD_DIR_SLASH; + } + path.append(name.begin(), name.size()); + SCOPE_EXIT { + path.resize(size); + }; + Status status; +#ifdef DT_DIR + if (entry->d_type == DT_UNKNOWN) { + status = walk_path(path, std::forward<Func>(func)); + } else if (entry->d_type == DT_DIR) { + status = walk_path_dir(path, std::forward<Func>(func)); + } else if (entry->d_type == DT_REG) { + status = walk_path_file(path, std::forward<Func>(func)); + } +#else +#warning "Slow walk_path" + status = walk_path(path, std::forward<Func>(func)); +#endif + if (status.is_error()) { + return status; + } + } +} + +template <class Func> +Status walk_path_dir(string &path, DIR *subdir, Func &&func) { + SCOPE_EXIT { + closedir(subdir); + }; + TRY_STATUS(walk_path_subdir(path, subdir, std::forward<Func>(func))); + std::forward<Func>(func)(path, true); + return Status::OK(); +} + +template <class Func> +Status walk_path_dir(string &path, FileFd fd, Func &&func) { + auto *subdir = fdopendir(fd.get_fd().move_as_native_fd()); + if (subdir == nullptr) { + auto error = OS_ERROR("fdopendir"); + fd.close(); + return error; + } + return walk_path_dir(path, subdir, std::forward<Func>(func)); +} + +template <class Func> +Status walk_path_dir(string &path, Func &&func) { + auto *subdir = opendir(path.c_str()); + if (subdir == nullptr) { + return OS_ERROR(PSLICE() << tag("opendir", path)); + } + return walk_path_dir(path, subdir, std::forward<Func>(func)); +} + +template <class Func> +Status walk_path_file(string &path, Func &&func) { + std::forward<Func>(func)(path, false); + return Status::OK(); +} + +template <class Func> +Status walk_path(string &path, Func &&func) { + TRY_RESULT(fd, FileFd::open(path, FileFd::Read)); + auto stat = fd.stat(); + bool is_dir = stat.is_dir_; + bool is_reg = stat.is_reg_; + if (is_dir) { + return walk_path_dir(path, std::move(fd), std::forward<Func>(func)); + } + + fd.close(); + if (is_reg) { + return walk_path_file(path, std::forward<Func>(func)); + } + + return Status::OK(); +} +} // namespace detail + +template <class Func> +Status walk_path(CSlice path, Func &&func) { + string curr_path; + curr_path.reserve(PATH_MAX + 10); + curr_path = path.c_str(); + return detail::walk_path(curr_path, std::forward<Func>(func)); +} + +#endif + +#if TD_PORT_WINDOWS + +namespace detail { +template <class Func> +Status walk_path_dir(const std::wstring &dir_name, Func &&func) { + std::wstring name = dir_name + L"\\*"; + + WIN32_FIND_DATA file_data; + auto handle = FindFirstFileExW(name.c_str(), FindExInfoStandard, &file_data, FindExSearchNameMatch, nullptr, 0); + if (handle == INVALID_HANDLE_VALUE) { + return OS_ERROR(PSLICE() << "FindFirstFileEx" << tag("name", from_wstring(name).ok())); + } + + SCOPE_EXIT { + FindClose(handle); + }; + while (true) { + auto full_name = dir_name + L"\\" + file_data.cFileName; + TRY_RESULT(entry_name, from_wstring(full_name)); + if (file_data.cFileName[0] != '.') { + if ((file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { + TRY_STATUS(walk_path_dir(full_name, func)); + func(entry_name, true); + } else { + func(entry_name, false); + } + } + auto status = FindNextFileW(handle, &file_data); + if (status == 0) { + auto last_error = GetLastError(); + if (last_error == ERROR_NO_MORE_FILES) { + return Status::OK(); + } + return OS_ERROR("FindNextFileW"); + } + } +} +} // namespace detail + +template <class Func> +Status walk_path(CSlice path, Func &&func) { + Slice path_slice = path; + while (!path_slice.empty() && (path_slice.back() == '/' || path_slice.back() == '\\')) { + path_slice.remove_suffix(1); + } + TRY_RESULT(wpath, to_wstring(path_slice)); + return detail::walk_path_dir(wpath.c_str(), func); +} + +#endif + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/platform.h b/libs/tdlib/td/tdutils/td/utils/port/platform.h new file mode 100644 index 0000000000..a1c3776a40 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/platform.h @@ -0,0 +1,106 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +// clang-format off + +/*** Platform macros ***/ +#if defined(_WIN32) + #if defined(__cplusplus_winrt) + #define TD_WINRT 1 + #endif + #if defined(__cplusplus_cli) + #define TD_CLI 1 + #endif + #define TD_WINDOWS 1 +#elif defined(__APPLE__) + #include "TargetConditionals.h" + #if TARGET_OS_IPHONE + // iOS/Apple Watch OS/Apple TV OS + #if TARGET_OS_IOS + #define TD_DARWIN_IOS 1 + #elif TARGET_OS_TV + #define TD_DARWIN_TV_OS 1 + #elif TARGET_OS_WATCH + #define TD_DARWIN_WATCH_OS 1 + #else + #warning "Probably unsupported Apple iPhone platform. Feel free to try to compile" + #endif + #elif TARGET_OS_MAC + // Other kinds of Mac OS + #define TD_DARWIN_MAC 1 + #else + #warning "Probably unsupported Apple platform. Feel free to try to compile" + #endif + #define TD_DARWIN 1 +#elif defined(ANDROID) || defined(__ANDROID__) + #define TD_ANDROID 1 +#elif defined(TIZEN_DEPRECATION) + #define TD_TIZEN 1 +#elif defined(__linux__) + #define TD_LINUX 1 +#elif defined(__CYGWIN__) + #define TD_CYGWIN 1 +#elif defined(__EMSCRIPTEN__) + #define TD_EMSCRIPTEN 1 +#elif defined(__unix__) // all unices not caught above + #warning "Probably unsupported Unix platform. Feel free to try to compile" + #define TD_CYGWIN 1 +#else + #error "Probably unsupported platform. Feel free to remove the error and try to recompile" +#endif + +#if defined(__ICC) || defined(__INTEL_COMPILER) + #define TD_INTEL 1 +#elif defined(__clang__) + #define TD_CLANG 1 +#elif defined(__GNUC__) || defined(__GNUG__) + #define TD_GCC 1 +#elif defined(_MSC_VER) + #define TD_MSVC 1 +#else + #warning "Probably unsupported compiler. Feel free to try to compile" +#endif + +#if TD_GCC || TD_CLANG || TD_INTEL + #define TD_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) + #define TD_ATTRIBUTE_FORMAT_PRINTF(from, to) __attribute__((format(printf, from, to))) +#else + #define TD_WARN_UNUSED_RESULT + #define TD_ATTRIBUTE_FORMAT_PRINTF(from, to) +#endif + +#if TD_MSVC + #define TD_UNUSED __pragma(warning(suppress : 4100)) +#elif TD_CLANG || TD_GCC || TD_INTEL + #define TD_UNUSED __attribute__((unused)) +#else + #define TD_UNUSED +#endif + +#define TD_HAVE_ATOMIC_SHARED_PTR 1 + +// No atomic operations on std::shared_ptr in libstdc++ before 5.0 +// see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=57250 +#ifdef __GLIBCXX__ + #undef TD_HAVE_ATOMIC_SHARED_PTR +#endif + +// Also no atomic operations on std::shared_ptr when clang __has_feature(cxx_atomic) is defined and zero +#if defined(__has_feature) + #if !__has_feature(cxx_atomic) + #undef TD_HAVE_ATOMIC_SHARED_PTR + #endif +#endif + +#ifdef TD_HAVE_ATOMIC_SHARED_PTR // unfortunately we can't check for __GLIBCXX__ here, it is not defined yet + #undef TD_HAVE_ATOMIC_SHARED_PTR +#endif + +#define TD_CONCURRENCY_PAD 128 + +// clang-format on diff --git a/libs/tdlib/td/tdutils/td/utils/port/signals.cpp b/libs/tdlib/td/tdutils/td/utils/port/signals.cpp new file mode 100644 index 0000000000..8627474d63 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/signals.cpp @@ -0,0 +1,298 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/signals.h" + +#include "td/utils/port/config.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" + +#if TD_PORT_POSIX +#include <signal.h> +#include <sys/mman.h> +#include <unistd.h> +#endif +#if TD_PORT_WINDOWS +#include <csignal> +#endif + +#include <cerrno> +#include <cstdint> +#include <cstring> +#include <ctime> +#include <limits> + +namespace td { + +#if TD_PORT_POSIX && !TD_DARWIN_TV_OS && !TD_DARWIN_WATCH_OS +static Status protect_memory(void *addr, size_t len) { + if (mprotect(addr, len, PROT_NONE) != 0) { + return OS_ERROR("mprotect failed"); + } + return Status::OK(); +} +#endif + +Status setup_signals_alt_stack() { +#if TD_PORT_POSIX && !TD_DARWIN_TV_OS && !TD_DARWIN_WATCH_OS + auto page_size = getpagesize(); + auto stack_size = (MINSIGSTKSZ + 16 * page_size - 1) / page_size * page_size; + + void *stack = mmap(nullptr, stack_size + 2 * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (stack == MAP_FAILED) { + return OS_ERROR("Mmap failed"); + } + + TRY_STATUS(protect_memory(stack, page_size)); + TRY_STATUS(protect_memory(static_cast<char *>(stack) + stack_size + page_size, page_size)); + + stack_t signal_stack; + signal_stack.ss_sp = static_cast<char *>(stack) + page_size; + signal_stack.ss_size = stack_size; + signal_stack.ss_flags = 0; + + if (sigaltstack(&signal_stack, nullptr) != 0) { + return OS_ERROR("sigaltstack failed"); + } +#endif + return Status::OK(); +} + +#if TD_PORT_POSIX +template <class F> +static Status set_signal_handler_impl(vector<int> signals, F func, bool is_extended = false) { + struct sigaction act; + std::memset(&act, '\0', sizeof(act)); + if (is_extended) { // TODO if constexpr, remove useless reinterpret_cast + act.sa_handler = reinterpret_cast<decltype(act.sa_handler)>(func); + } else { + act.sa_sigaction = reinterpret_cast<decltype(act.sa_sigaction)>(func); + } + sigemptyset(&act.sa_mask); + for (auto signal : signals) { + sigaddset(&act.sa_mask, signal); + } + act.sa_flags = SA_RESTART | SA_ONSTACK; + if (is_extended) { + act.sa_flags |= SA_SIGINFO; + } + + for (auto signal : signals) { + if (sigaction(signal, &act, nullptr) != 0) { + return OS_ERROR("sigaction failed"); + } + } + return Status::OK(); +} + +static vector<int> get_native_signals(SignalType type) { + switch (type) { + case SignalType::Abort: + return {SIGABRT, SIGXCPU, SIGXFSZ}; + case SignalType::Error: + return {SIGILL, SIGFPE, SIGBUS, SIGSEGV, SIGSYS}; + case SignalType::Quit: + return {SIGINT, SIGTERM, SIGQUIT}; + case SignalType::Pipe: + return {SIGPIPE}; + case SignalType::HangUp: + return {SIGHUP}; + case SignalType::User: + return {SIGUSR1, SIGUSR2}; + case SignalType::Other: + return {SIGTRAP, SIGALRM, SIGVTALRM, SIGPROF, SIGTSTP, SIGTTIN, SIGTTOU}; + default: + return {}; + } +} +#endif +#if TD_PORT_WINDOWS +static Status set_signal_handler_impl(vector<int> signals, void (*func)(int sig), bool /*unused*/ = true) { + for (auto signal : signals) { + if (std::signal(signal, func) == SIG_ERR) { + return Status::Error("Failed to set signal handler"); + } + } + return Status::OK(); +} + +static vector<int> get_native_signals(SignalType type) { + switch (type) { + case SignalType::Abort: + return {SIGABRT}; + case SignalType::Error: + return {SIGILL, SIGFPE, SIGSEGV}; + case SignalType::Quit: + return {SIGINT, SIGTERM}; + case SignalType::Pipe: + return {}; + case SignalType::HangUp: + return {}; + case SignalType::User: + return {}; + case SignalType::Other: + return {}; + default: + return {}; + } +} +#endif + +Status set_signal_handler(SignalType type, void (*func)(int)) { + return set_signal_handler_impl(get_native_signals(type), func == nullptr ? SIG_DFL : func); +} + +using extended_signal_handler = void (*)(int sig, void *addr); +static extended_signal_handler extended_signal_handlers[NSIG] = {}; + +#if TD_PORT_POSIX +static void siginfo_handler(int signum, siginfo_t *info, void *data) { + auto handler = extended_signal_handlers[signum]; + handler(signum, info->si_addr); +} +#elif TD_PORT_WINDOWS +static void siginfo_handler(int signum) { + auto handler = extended_signal_handlers[signum]; + handler(signum, nullptr); +} +#endif + +Status set_extended_signal_handler(SignalType type, extended_signal_handler func) { + CHECK(func != nullptr); + auto signals = get_native_signals(type); + for (auto signal : signals) { + if (0 <= signal && signal < NSIG) { + extended_signal_handlers[signal] = func; + } else { + UNREACHABLE(); + } + } + return set_signal_handler_impl(std::move(signals), siginfo_handler, true); +} + +Status set_runtime_signal_handler(int runtime_signal_number, void (*func)(int)) { +#ifdef SIGRTMIN + CHECK(SIGRTMIN + runtime_signal_number <= SIGRTMAX); + return set_signal_handler_impl({SIGRTMIN + runtime_signal_number}, func == nullptr ? SIG_DFL : func); +#else + return Status::OK(); +#endif +} + +Status ignore_signal(SignalType type) { + return set_signal_handler_impl(get_native_signals(type), SIG_IGN); +} + +static void signal_safe_append_int(char **s, Slice name, int number) { + if (number < 0) { + number = std::numeric_limits<int>::max(); + } + + *--*s = ' '; + *--*s = ']'; + + do { + *--*s = static_cast<char>(number % 10 + '0'); + number /= 10; + } while (number > 0); + + *--*s = ' '; + + for (auto pos = static_cast<int>(name.size()) - 1; pos >= 0; pos--) { + *--*s = name[pos]; + } + + *--*s = '['; +} + +static void signal_safe_write_data(Slice data) { +#if TD_PORT_POSIX + while (!data.empty()) { + auto res = write(2, data.begin(), data.size()); + if (res < 0 && errno == EINTR) { + continue; + } + if (res <= 0) { + break; + } + + if (res > 0) { + data.remove_prefix(res); + } + } +#elif TD_PORT_WINDOWS +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) + HANDLE stderr_handle = GetStdHandle(STD_ERROR_HANDLE); + DWORD bytes_written; + WriteFile(stderr_handle, data.data(), static_cast<DWORD>(data.size()), &bytes_written, nullptr); +#else +// there is no stderr +#endif +#endif +} + +static int get_process_id() { +#if TD_PORT_POSIX + return getpid(); +#elif TD_PORT_WINDOWS + return GetCurrentProcessId(); +#endif +} + +void signal_safe_write(Slice data, bool add_header) { + auto old_errno = errno; + + if (add_header) { + constexpr size_t HEADER_BUF_SIZE = 100; + char header[HEADER_BUF_SIZE]; + char *header_end = header + HEADER_BUF_SIZE; + char *header_begin = header_end; + + signal_safe_append_int(&header_begin, "time", static_cast<int>(std::time(nullptr))); + signal_safe_append_int(&header_begin, "pid", get_process_id()); + + signal_safe_write_data(Slice(header_begin, header_end)); + } + + signal_safe_write_data(data); + + errno = old_errno; +} + +void signal_safe_write_signal_number(int sig, bool add_header) { + char buf[100]; + char *end = buf + sizeof(buf); + char *ptr = end; + *--ptr = '\n'; + do { + *--ptr = static_cast<char>(sig % 10 + '0'); + sig /= 10; + } while (sig != 0); + + ptr -= 8; + std::memcpy(ptr, "Signal: ", 8); + signal_safe_write(Slice(ptr, end), add_header); +} + +void signal_safe_write_pointer(void *p, bool add_header) { + std::uintptr_t addr = reinterpret_cast<std::uintptr_t>(p); + char buf[100]; + char *end = buf + sizeof(buf); + char *ptr = end; + *--ptr = '\n'; + do { + *--ptr = td::format::hex_digit(addr % 16); + addr /= 16; + } while (addr != 0); + *--ptr = 'x'; + *--ptr = '0'; + ptr -= 9; + std::memcpy(ptr, "Address: ", 9); + signal_safe_write(Slice(ptr, end), add_header); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/signals.h b/libs/tdlib/td/tdutils/td/utils/port/signals.h new file mode 100644 index 0000000000..1f6ed24732 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/signals.h @@ -0,0 +1,34 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +Status setup_signals_alt_stack() TD_WARN_UNUSED_RESULT; + +enum class SignalType { Abort, Error, Quit, Pipe, HangUp, User, Other }; + +Status set_signal_handler(SignalType type, void (*func)(int sig)) TD_WARN_UNUSED_RESULT; + +Status set_extended_signal_handler(SignalType type, void (*func)(int sig, void *addr)) TD_WARN_UNUSED_RESULT; + +Status set_runtime_signal_handler(int runtime_signal_number, void (*func)(int sig)) TD_WARN_UNUSED_RESULT; + +Status ignore_signal(SignalType type) TD_WARN_UNUSED_RESULT; + +// writes data to the standard error stream in a signal-safe way +void signal_safe_write(Slice data, bool add_header = true); + +void signal_safe_write_signal_number(int sig, bool add_header = true); + +void signal_safe_write_pointer(void *p, bool add_header = true); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/sleep.cpp b/libs/tdlib/td/tdutils/td/utils/port/sleep.cpp new file mode 100644 index 0000000000..4f02f69dfb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/sleep.cpp @@ -0,0 +1,37 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/sleep.h" + +#include "td/utils/port/config.h" + +#if TD_PORT_POSIX +#if _POSIX_C_SOURCE >= 199309L +#include <time.h> +#else +#include <unistd.h> +#endif +#endif + +namespace td { + +void usleep_for(int32 microseconds) { +#if TD_PORT_WINDOWS + int32 milliseconds = microseconds / 1000 + (microseconds % 1000 ? 1 : 0); + Sleep(milliseconds); +#else +#if _POSIX_C_SOURCE >= 199309L + timespec ts; + ts.tv_sec = microseconds / 1000000; + ts.tv_nsec = (microseconds % 1000000) * 1000; + nanosleep(&ts, nullptr); +#else + usleep(microseconds); +#endif +#endif +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/sleep.h b/libs/tdlib/td/tdutils/td/utils/port/sleep.h new file mode 100644 index 0000000000..56cd9f7bcd --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/sleep.h @@ -0,0 +1,15 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +namespace td { + +void usleep_for(int32 microseconds); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/thread.h b/libs/tdlib/td/tdutils/td/utils/port/thread.h new file mode 100644 index 0000000000..3034e456e8 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/thread.h @@ -0,0 +1,34 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/port/detail/ThreadPthread.h" +#include "td/utils/port/detail/ThreadStl.h" + +namespace td { + +// clang-format off + +#if TD_THREAD_PTHREAD + using thread = detail::ThreadPthread; + namespace this_thread = detail::this_thread_pthread; +#elif TD_THREAD_STL + using thread = detail::ThreadStl; + namespace this_thread = detail::this_thread_stl; +#elif TD_THREAD_UNSUPPORTED + namespace this_thread { + inline void yield() {} + } +#else + #error "Thread's implementation is not defined" +#endif + +// clang-format on + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/thread_local.cpp b/libs/tdlib/td/tdutils/td/utils/port/thread_local.cpp new file mode 100644 index 0000000000..aa4e371405 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/thread_local.cpp @@ -0,0 +1,41 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/thread_local.h" + +#include "td/utils/logging.h" + +namespace td { + +namespace detail { + +static TD_THREAD_LOCAL int32 thread_id_; +static TD_THREAD_LOCAL std::vector<std::unique_ptr<Guard>> *thread_local_destructors; + +void add_thread_local_destructor(std::unique_ptr<Guard> destructor) { + if (thread_local_destructors == nullptr) { + thread_local_destructors = new std::vector<std::unique_ptr<Guard>>(); + } + thread_local_destructors->push_back(std::move(destructor)); +} + +} // namespace detail + +void clear_thread_locals() { + // ensure that no destructors were added during destructors invokation + auto to_delete = detail::thread_local_destructors; + detail::thread_local_destructors = nullptr; + delete to_delete; + CHECK(detail::thread_local_destructors == nullptr); +} +void set_thread_id(int32 id) { + detail::thread_id_ = id; +} + +int32 get_thread_id() { + return detail::thread_id_; +} +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/thread_local.h b/libs/tdlib/td/tdutils/td/utils/port/thread_local.h new file mode 100644 index 0000000000..6d8c135e88 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/thread_local.h @@ -0,0 +1,69 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#include "td/utils/common.h" +#include "td/utils/ScopeGuard.h" + +#include <memory> +#include <utility> + +namespace td { + +// clang-format off +#if TD_GCC || TD_CLANG + #define TD_THREAD_LOCAL __thread +#elif TD_INTEL || TD_MSVC + #define TD_THREAD_LOCAL thread_local +#else + #warning "TD_THREAD_LOCAL is not defined, trying 'thread_local'" + #define TD_THREAD_LOCAL thread_local +#endif +// clang-format on + +inline constexpr size_t max_thread_count() { + return 256; +} + +// If raw_ptr is not nullptr, allocate T as in std::make_unique<T>(args...) and store pointer into raw_ptr +template <class T, class P, class... ArgsT> +bool init_thread_local(P &raw_ptr, ArgsT &&... args); + +// Destroy all thread locals, and store nullptr into corresponding pointers +void clear_thread_locals(); + +void set_thread_id(int32 id); + +int32 get_thread_id(); + +namespace detail { +void add_thread_local_destructor(std::unique_ptr<Guard> destructor); + +template <class T, class P, class... ArgsT> +void do_init_thread_local(P &raw_ptr, ArgsT &&... args) { + auto ptr = std::make_unique<T>(std::forward<ArgsT>(args)...); + raw_ptr = ptr.get(); + + detail::add_thread_local_destructor(create_lambda_guard([ptr = std::move(ptr), &raw_ptr]() mutable { + ptr.reset(); + raw_ptr = nullptr; + })); +} +} // namespace detail + +template <class T, class P, class... ArgsT> +bool init_thread_local(P &raw_ptr, ArgsT &&... args) { + if (likely(raw_ptr != nullptr)) { + return false; + } + detail::do_init_thread_local<T>(raw_ptr, std::forward<ArgsT>(args)...); + return true; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/port/wstring_convert.cpp b/libs/tdlib/td/tdutils/td/utils/port/wstring_convert.cpp new file mode 100644 index 0000000000..56da62b96d --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/wstring_convert.cpp @@ -0,0 +1,63 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/port/wstring_convert.h" + +char disable_linker_warning_about_empty_file_wstring_convert_cpp TD_UNUSED; + +#if TD_PORT_WINDOWS + +#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING + +#include "td/utils/port/wstring_convert.h" + +#include <codecvt> +#include <locale> +#include <utility> + +namespace td { + +namespace detail { +template <class Facet> +class UsableFacet : public Facet { + public: + template <class... Args> + explicit UsableFacet(Args &&... args) : Facet(std::forward<Args>(args)...) { + } + ~UsableFacet() = default; +}; +} // namespace detail + +Result<std::wstring> to_wstring(Slice slice) { + // TODO(perf): optimize + std::wstring_convert<detail::UsableFacet<std::codecvt_utf8_utf16<wchar_t>>> converter; + auto res = converter.from_bytes(slice.begin(), slice.end()); + if (converter.converted() != slice.size()) { + return Status::Error("Wrong encoding"); + } + return res; +} + +Result<string> from_wstring(const wchar_t *begin, size_t size) { + std::wstring_convert<detail::UsableFacet<std::codecvt_utf8_utf16<wchar_t>>> converter; + auto res = converter.to_bytes(begin, begin + size); + if (converter.converted() != size) { + return Status::Error("Wrong encoding"); + } + return res; +} + +Result<string> from_wstring(const std::wstring &str) { + return from_wstring(str.data(), str.size()); +} + +Result<string> from_wstring(const wchar_t *begin) { + return from_wstring(begin, wcslen(begin)); +} + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/port/wstring_convert.h b/libs/tdlib/td/tdutils/td/utils/port/wstring_convert.h new file mode 100644 index 0000000000..a795d2bd92 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/port/wstring_convert.h @@ -0,0 +1,31 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/config.h" + +#if TD_PORT_WINDOWS + +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <string> + +namespace td { + +Result<std::wstring> to_wstring(Slice slice); + +Result<string> from_wstring(const std::wstring &str); + +Result<string> from_wstring(const wchar_t *begin, size_t size); + +Result<string> from_wstring(const wchar_t *begin); + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/queue.h b/libs/tdlib/td/tdutils/td/utils/queue.h new file mode 100644 index 0000000000..6d107e37f2 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/queue.h @@ -0,0 +1,484 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/EventFd.h" +#include "td/utils/port/thread.h" + +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED + +#if !TD_WINDOWS +#include <poll.h> +#include <sched.h> +#endif + +#include <atomic> +#include <type_traits> +#include <utility> + +namespace td { + +namespace detail { +class Backoff { + int cnt = 0; + + public: + bool next() { + // TODO: find out better strategy + // TODO: try adaptive backoff + // TODO: different strategy one core cpu + // return false; + + cnt++; + if (cnt < 1) { // 50 + return true; + } else { + td::this_thread::yield(); + return cnt < 3; // 500 + } + } +}; + +class InfBackoff { + int cnt = 0; + + public: + bool next() { + cnt++; + if (cnt < 50) { + return true; + } else { + td::this_thread::yield(); + return true; + } + } +}; + +} // namespace detail + +template <class T, int P = 10> +class SPSCBlockQueue { + public: + using ValueType = T; + + private: + static constexpr int buffer_size() { + static_assert(P >= 1 && P <= 20, "Bad size of BlockQueue"); + return 1 << P; + } + + struct Position { + std::atomic<uint32> i; + char pad[64 - sizeof(std::atomic<uint32>)]; + uint32 local_writer_i; + char pad2[64 - sizeof(uint32)]; + uint32 local_reader_i; + char pad3[64 - sizeof(uint32)]; + + void init() { + i = 0; + local_reader_i = 0; + local_writer_i = 0; + } + }; + + typename std::aligned_storage<sizeof(ValueType)>::type data_[buffer_size()]; + Position writer_; + Position reader_; + + static int fix_i(int i) { + return i & (buffer_size() - 1); + } + + ValueType *at_ptr(int i) { + return reinterpret_cast<ValueType *>(&data_[fix_i(i)]); + } + + ValueType &at(int i) { + return *at_ptr(i); + } + + public: + void init() { + writer_.init(); + reader_.init(); + } + + void destroy() { + } + + int writer_size() { + return static_cast<int>(writer_.local_reader_i + buffer_size() - writer_.local_writer_i); + } + + bool writer_empty() { + return writer_.local_reader_i + buffer_size() == writer_.local_writer_i; + } + + template <class PutValueType> + void writer_put_unsafe(PutValueType &&value) { + at(writer_.local_writer_i++) = std::forward<PutValueType>(value); + } + + int writer_update() { + writer_.local_reader_i = reader_.i.load(std::memory_order_acquire); + return writer_size(); + } + + void writer_flush() { + writer_.i.store(writer_.local_writer_i, std::memory_order_release); + } + + int reader_size() { + return static_cast<int>(reader_.local_writer_i - reader_.local_reader_i); + } + + int reader_empty() { + return reader_.local_writer_i == reader_.local_reader_i; + } + + ValueType reader_get_unsafe() { + return std::move(at(reader_.local_reader_i++)); + } + + int reader_update() { + reader_.local_writer_i = writer_.i.load(std::memory_order_acquire); + return reader_size(); + } + + void reader_flush() { + reader_.i.store(reader_.local_reader_i, std::memory_order_release); + } +}; + +template <class T, class BlockQueueT = SPSCBlockQueue<T> > +class SPSCChainQueue { + public: + using ValueType = T; + + void init() { + head_ = tail_ = create_node(); + } + + SPSCChainQueue() = default; + SPSCChainQueue(const SPSCChainQueue &) = delete; + SPSCChainQueue &operator=(const SPSCChainQueue &) = delete; + SPSCChainQueue(SPSCChainQueue &&) = delete; + SPSCChainQueue &operator=(SPSCChainQueue &&) = delete; + ~SPSCChainQueue() { + destroy(); + } + + void destroy() { + while (head_ != nullptr) { + Node *to_delete = head_; + head_ = head_->next_; + delete_node(to_delete); + } + tail_ = nullptr; + } + + int writer_size() { + return tail_->q_.writer_size(); + } + + bool writer_empty() { + return tail_->q_.writer_empty(); + } + + template <class PutValueType> + void writer_put_unsafe(PutValueType &&value) { + tail_->q_.writer_put_unsafe(std::forward<PutValueType>(value)); + } + + int writer_update() { + int res = tail_->q_.writer_update(); + if (res != 0) { + return res; + } + + writer_flush(); + + Node *new_tail = create_node(); + tail_->next_ = new_tail; + tail_->is_closed_.store(true, std::memory_order_release); + tail_ = new_tail; + return tail_->q_.writer_update(); + } + + void writer_flush() { + tail_->q_.writer_flush(); + } + + int reader_size() { + return head_->q_.reader_size(); + } + + int reader_empty() { + return head_->q_.reader_empty(); + } + + ValueType reader_get_unsafe() { + return std::move(head_->q_.reader_get_unsafe()); + } + + int reader_update() { + int res = head_->q_.reader_update(); + if (res != 0) { + return res; + } + + if (!head_->is_closed_.load(std::memory_order_acquire)) { + return 0; + } + + res = head_->q_.reader_update(); + if (res != 0) { + return res; + } + + // reader_flush(); + + Node *old_head = head_; + head_ = head_->next_; + delete_node(old_head); + + return head_->q_.reader_update(); + } + + void reader_flush() { + head_->q_.reader_flush(); + } + + private: + struct Node { + BlockQueueT q_; + std::atomic<bool> is_closed_; + Node *next_; + + void init() { + q_.init(); + is_closed_ = false; + next_ = nullptr; + } + + void destroy() { + q_.destroy(); + next_ = nullptr; + } + }; + + Node *head_; + char pad[64 - sizeof(Node *)]; + Node *tail_; + char pad2[64 - sizeof(Node *)]; + + Node *create_node() { + Node *res = new Node(); + res->init(); + return res; + } + + void delete_node(Node *node) { + node->destroy(); + delete node; + } +}; + +template <class T, class QueueT = SPSCChainQueue<T>, class BackoffT = detail::Backoff> +class BackoffQueue : public QueueT { + public: + using ValueType = T; + + template <class PutValueType> + void writer_put(PutValueType &&value) { + if (this->writer_empty()) { + int sz = this->writer_update(); + CHECK(sz != 0); + } + this->writer_put_unsafe(std::forward<PutValueType>(value)); + } + + int reader_wait() { + BackoffT backoff; + int res = 0; + do { + res = this->reader_update(); + } while (res == 0 && backoff.next()); + return res; + } +}; + +template <class T, class QueueT = SPSCChainQueue<T> > +using InfBackoffQueue = BackoffQueue<T, QueueT, detail::InfBackoff>; + +template <class T, class QueueT = BackoffQueue<T> > +class PollQueue : public QueueT { + public: + using ValueType = T; + using QueueType = QueueT; + + void init() { + QueueType::init(); + event_fd_.init(); + wait_state_ = 0; + writer_wait_state_ = 0; + } + + PollQueue() = default; + PollQueue(const PollQueue &) = delete; + PollQueue &operator=(const PollQueue &) = delete; + PollQueue(PollQueue &&) = delete; + PollQueue &operator=(PollQueue &&) = delete; + ~PollQueue() { + destroy_impl(); + } + void destroy() { + destroy_impl(); + QueueType::destroy(); + } + + void writer_flush() { + int old_wait_state = get_wait_state(); + + std::atomic_thread_fence(std::memory_order_seq_cst); + + QueueType::writer_flush(); + + std::atomic_thread_fence(std::memory_order_seq_cst); + + int wait_state = get_wait_state(); + if ((wait_state & 1) && wait_state != writer_wait_state_) { + event_fd_.release(); + writer_wait_state_ = old_wait_state; + } + } + + EventFd &reader_get_event_fd() { + return event_fd_; + } + + // if 0 is returned than it is useless to rerun it before fd is + // ready to read. + int reader_wait_nonblock() { + int res; + + if ((get_wait_state() & 1) == 0) { + res = this->QueueType::reader_wait(); + if (res != 0) { + return res; + } + + inc_wait_state(); + + std::atomic_thread_fence(std::memory_order_seq_cst); + + res = this->reader_update(); + if (res != 0) { + inc_wait_state(); + return res; + } + } + + event_fd_.acquire(); + std::atomic_thread_fence(std::memory_order_seq_cst); + res = this->reader_update(); + if (res != 0) { + inc_wait_state(); + } + return res; + } + +// Just example of usage +#if !TD_WINDOWS + int reader_wait() { + int res; + + while ((res = reader_wait_nonblock()) == 0) { + // TODO: reader_flush? + pollfd fd; + fd.fd = reader_get_event_fd().get_fd().get_native_fd(); + fd.events = POLLIN; + poll(&fd, 1, -1); + } + return res; + } +#endif + + private: + EventFd event_fd_; + std::atomic<int> wait_state_; + int writer_wait_state_; + + int get_wait_state() { + return wait_state_.load(std::memory_order_relaxed); + } + + void inc_wait_state() { + wait_state_.store(get_wait_state() + 1, std::memory_order_relaxed); + } + + void destroy_impl() { + if (!event_fd_.empty()) { + event_fd_.close(); + } + } +}; + +} // namespace td + +#else + +#include "td/utils/logging.h" + +namespace td { + +// dummy implementation which shouldn't be used + +template <class T> +class PollQueue { + public: + using ValueType = T; + + void init() { + UNREACHABLE(); + } + + template <class PutValueType> + void writer_put(PutValueType &&value) { + UNREACHABLE(); + } + + void writer_flush() { + UNREACHABLE(); + } + + int reader_wait_nonblock() { + UNREACHABLE(); + return 0; + } + + ValueType reader_get_unsafe() { + UNREACHABLE(); + return ValueType(); + } + + void reader_flush() { + UNREACHABLE(); + } + + PollQueue() = default; + PollQueue(const PollQueue &) = delete; + PollQueue &operator=(const PollQueue &) = delete; + PollQueue(PollQueue &&) = delete; + PollQueue &operator=(PollQueue &&) = delete; + ~PollQueue() = default; +}; + +} // namespace td + +#endif diff --git a/libs/tdlib/td/tdutils/td/utils/tests.h b/libs/tdlib/td/tdutils/td/utils/tests.h new file mode 100644 index 0000000000..24e2f3fe22 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/tests.h @@ -0,0 +1,205 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/List.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/Time.h" + +#include <atomic> + +#define REGISTER_TESTS(x) \ + void TD_CONCAT(register_tests_, x)() { \ + } +#define DESC_TESTS(x) void TD_CONCAT(register_tests_, x)() +#define LOAD_TESTS(x) TD_CONCAT(register_tests_, x)() + +namespace td { + +class Test : private ListNode { + public: + explicit Test(CSlice name) : name_(name) { + get_tests_list()->put_back(this); + } + + Test(const Test &) = delete; + Test &operator=(const Test &) = delete; + Test(Test &&) = delete; + Test &operator=(Test &&) = delete; + virtual ~Test() = default; + + static void add_substr_filter(std::string str) { + if (str[0] != '+' && str[0] != '-') { + str = "+" + str; + } + get_substr_filters()->push_back(std::move(str)); + } + static void set_stress_flag(bool flag) { + get_stress_flag() = flag; + } + static void run_all() { + while (run_all_step()) { + } + } + + static bool run_all_step() { + auto *state = get_state(); + if (state->it == nullptr) { + state->end = get_tests_list(); + state->it = state->end->next; + } + + while (state->it != state->end) { + auto test = static_cast<td::Test *>(state->it); + if (!state->is_running) { + bool ok = true; + for (const auto &filter : *get_substr_filters()) { + bool is_match = test->name_.str().find(filter.substr(1)) != std::string::npos; + if (is_match != (filter[0] == '+')) { + ok = false; + break; + } + } + if (!ok) { + state->it = state->it->next; + continue; + } + LOG(ERROR) << "Run test " << tag("name", test->name_); + state->start = Time::now(); + state->is_running = true; + } + + if (test->step()) { + break; + } + + LOG(ERROR) << format::as_time(Time::now() - state->start); + state->is_running = false; + state->it = state->it->next; + } + + auto ret = state->it != state->end; + if (!ret) { + *state = State(); + } + return ret || get_stress_flag(); + } + + private: + CSlice name_; + struct State { + ListNode *it = nullptr; + bool is_running = false; + double start; + ListNode *end = nullptr; + }; + static State *get_state() { + static State state; + return &state; + } + static std::vector<std::string> *get_substr_filters() { + static std::vector<std::string> substr_filters_; + return &substr_filters_; + } + + static ListNode *get_tests_list() { + static ListNode root; + return &root; + } + static bool &get_ok_flag() { + static bool is_ok = true; + return is_ok; + } + static bool &get_stress_flag() { + static bool stress_flag = false; + return stress_flag; + } + virtual void run() { + while (step()) { + } + } + + virtual bool step() { + run(); + return false; + } +}; + +class Stage { + public: + void wait(uint64 need) { + value_.fetch_add(1, std::memory_order_release); + while (value_.load(std::memory_order_acquire) < need) { + td::this_thread::yield(); + } + }; + + private: + std::atomic<uint64> value_{0}; +}; + +inline string rand_string(char from, char to, int len) { + string res(len, 0); + for (auto &c : res) { + c = static_cast<char>(Random::fast(from, to)); + } + return res; +} + +inline std::vector<string> rand_split(string str) { + std::vector<string> res; + size_t pos = 0; + while (pos < str.size()) { + size_t len; + if (Random::fast(0, 1) == 1) { + len = Random::fast(1, 10); + } else { + len = Random::fast(100, 200); + } + res.push_back(str.substr(pos, len)); + pos += len; + } + return res; +} + +template <class T1, class T2> +void assert_eq_impl(const T1 &expected, const T2 &got, const char *file, int line) { + CHECK(expected == got) << tag("expected", expected) << tag("got", got) << " in " << file << " at line " << line; +} + +template <class T> +void assert_true_impl(const T &got, const char *file, int line) { + CHECK(got) << "Expected true in " << file << " at line " << line; +} + +} // namespace td + +#define ASSERT_EQ(expected, got) ::td::assert_eq_impl((expected), (got), __FILE__, __LINE__) + +#define ASSERT_TRUE(got) ::td::assert_true_impl((got), __FILE__, __LINE__) + +#define ASSERT_STREQ(expected, got) \ + ::td::assert_eq_impl(::td::Slice((expected)), ::td::Slice((got)), __FILE__, __LINE__) + +#define TEST_NAME(test_case_name, test_name) \ + TD_CONCAT(Test, TD_CONCAT(_, TD_CONCAT(test_case_name, TD_CONCAT(_, test_name)))) + +#define TEST(test_case_name, test_name) TEST_IMPL(TEST_NAME(test_case_name, test_name)) + +#define TEST_IMPL(test_name) \ + class test_name : public ::td::Test { \ + public: \ + using Test::Test; \ + void run() final; \ + }; \ + test_name TD_CONCAT(test_instance_, TD_CONCAT(test_name, __LINE__))(TD_DEFINE_STR(test_name)); \ + void test_name::run() diff --git a/libs/tdlib/td/tdutils/td/utils/tl_helpers.h b/libs/tdlib/td/tdutils/td/utils/tl_helpers.h new file mode 100644 index 0000000000..686dacbeef --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/tl_helpers.h @@ -0,0 +1,203 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/Status.h" +#include "td/utils/tl_parsers.h" +#include "td/utils/tl_storers.h" + +#include <type_traits> +#include <unordered_set> + +#define BEGIN_STORE_FLAGS() \ + uint32 flags_store = 0; \ + uint32 bit_offset_store = 0 + +#define STORE_FLAG(flag) \ + flags_store |= (flag) << bit_offset_store; \ + bit_offset_store++ + +#define END_STORE_FLAGS() \ + CHECK(bit_offset_store < 31); \ + td::store(flags_store, storer) + +#define BEGIN_PARSE_FLAGS() \ + uint32 flags_parse; \ + uint32 bit_offset_parse = 0; \ + td::parse(flags_parse, parser) + +#define PARSE_FLAG(flag) \ + flag = ((flags_parse >> bit_offset_parse) & 1) != 0; \ + bit_offset_parse++ + +#define END_PARSE_FLAGS() \ + CHECK(bit_offset_parse < 31); \ + CHECK((flags_parse & ~((1 << bit_offset_parse) - 1)) == 0) << flags_parse << " " << bit_offset_parse; + +namespace td { +template <class StorerT> +void store(bool x, StorerT &storer) { + storer.store_binary(static_cast<int32>(x)); +} +template <class ParserT> +void parse(bool &x, ParserT &parser) { + x = parser.fetch_int() != 0; +} + +template <class StorerT> +void store(int32 x, StorerT &storer) { + storer.store_binary(x); +} +template <class ParserT> +void parse(int32 &x, ParserT &parser) { + x = parser.fetch_int(); +} + +template <class StorerT> +void store(uint32 x, StorerT &storer) { + storer.store_binary(x); +} +template <class ParserT> +void parse(uint32 &x, ParserT &parser) { + x = static_cast<uint32>(parser.fetch_int()); +} + +template <class StorerT> +void store(int64 x, StorerT &storer) { + storer.store_binary(x); +} +template <class ParserT> +void parse(int64 &x, ParserT &parser) { + x = parser.fetch_long(); +} +template <class StorerT> +void store(uint64 x, StorerT &storer) { + storer.store_binary(x); +} +template <class ParserT> +void parse(uint64 &x, ParserT &parser) { + x = static_cast<uint64>(parser.fetch_long()); +} + +template <class StorerT> +void store(double x, StorerT &storer) { + storer.store_binary(x); +} +template <class ParserT> +void parse(double &x, ParserT &parser) { + x = parser.fetch_double(); +} + +template <class StorerT> +void store(Slice x, StorerT &storer) { + storer.store_string(x); +} +template <class StorerT> +void store(const string &x, StorerT &storer) { + storer.store_string(x); +} +template <class ParserT> +void parse(string &x, ParserT &parser) { + x = parser.template fetch_string<string>(); +} + +template <class T, class StorerT> +void store(const vector<T> &vec, StorerT &storer) { + storer.store_binary(narrow_cast<int32>(vec.size())); + for (auto &val : vec) { + store(val, storer); + } +} +template <class T, class ParserT> +void parse(vector<T> &vec, ParserT &parser) { + uint32 size = parser.fetch_int(); + if (parser.get_left_len() < size) { + parser.set_error("Wrong vector length"); + return; + } + vec = vector<T>(size); + for (auto &val : vec) { + parse(val, parser); + } +} + +template <class Key, class Hash, class KeyEqual, class Allocator, class StorerT> +void store(const std::unordered_set<Key, Hash, KeyEqual, Allocator> &s, StorerT &storer) { + storer.store_binary(narrow_cast<int32>(s.size())); + for (auto &val : s) { + store(val, storer); + } +} +template <class Key, class Hash, class KeyEqual, class Allocator, class ParserT> +void parse(std::unordered_set<Key, Hash, KeyEqual, Allocator> &s, ParserT &parser) { + uint32 size = parser.fetch_int(); + if (parser.get_left_len() < size) { + parser.set_error("Wrong set length"); + return; + } + s.clear(); + Key val; + for (uint32 i = 0; i < size; i++) { + parse(val, parser); + s.insert(std::move(val)); + } +} + +template <class T, class StorerT> +std::enable_if_t<std::is_enum<T>::value> store(const T &val, StorerT &storer) { + store(static_cast<int32>(val), storer); +} +template <class T, class ParserT> +std::enable_if_t<std::is_enum<T>::value> parse(T &val, ParserT &parser) { + int32 result; + parse(result, parser); + val = static_cast<T>(result); +} + +template <class T, class StorerT> +std::enable_if_t<!std::is_enum<T>::value> store(const T &val, StorerT &storer) { + val.store(storer); +} +template <class T, class ParserT> +std::enable_if_t<!std::is_enum<T>::value> parse(T &val, ParserT &parser) { + val.parse(parser); +} + +template <class T> +string serialize(const T &object) { + TlStorerCalcLength calc_length; + store(object, calc_length); + size_t length = calc_length.get_length(); + + string key(length, '\0'); + if (!is_aligned_pointer<4>(key.data())) { + auto ptr = StackAllocator::alloc(length); + MutableSlice data = ptr.as_slice(); + TlStorerUnsafe storer(data.begin()); + store(object, storer); + key.assign(data.begin(), data.size()); + } else { + MutableSlice data = key; + TlStorerUnsafe storer(data.begin()); + store(object, storer); + } + return key; +} + +template <class T> +TD_WARN_UNUSED_RESULT Status unserialize(T &object, Slice data) { + TlParser parser(data); + parse(object, parser); + parser.fetch_end(); + return parser.get_status(); +} +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/tl_parsers.cpp b/libs/tdlib/td/tdutils/td/utils/tl_parsers.cpp new file mode 100644 index 0000000000..534e7793cf --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/tl_parsers.cpp @@ -0,0 +1,29 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/tl_parsers.h" + +namespace td { + +alignas(4) const unsigned char TlParser::empty_data[sizeof(UInt256)] = {}; // static zero-initialized + +void TlParser::set_error(const string &error_message) { + if (error.empty()) { + CHECK(!error_message.empty()); + error = error_message; + error_pos = data_len - left_len; + data = empty_data; + left_len = 0; + data_len = 0; + } else { + data = empty_data; + CHECK(error_pos != std::numeric_limits<size_t>::max()); + CHECK(data_len == 0); + CHECK(left_len == 0); + } +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/tl_parsers.h b/libs/tdlib/td/tdutils/td/utils/tl_parsers.h new file mode 100644 index 0000000000..ffb669bdeb --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/tl_parsers.h @@ -0,0 +1,242 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/utf8.h" + +#include <array> +#include <cstring> +#include <limits> +#include <string> + +namespace td { + +class TlParser { + const unsigned char *data = nullptr; + size_t data_len = 0; + size_t left_len = 0; + size_t error_pos = std::numeric_limits<size_t>::max(); + std::string error; + + unique_ptr<int32[]> data_buf; + static constexpr size_t SMALL_DATA_ARRAY_SIZE = 6; + std::array<int32, SMALL_DATA_ARRAY_SIZE> small_data_array; + + alignas(4) static const unsigned char empty_data[sizeof(UInt256)]; + + public: + explicit TlParser(Slice slice) { + if (slice.size() % sizeof(int32) != 0) { + set_error("Wrong length"); + return; + } + + data_len = left_len = slice.size(); + if (is_aligned_pointer<4>(slice.begin())) { + data = slice.ubegin(); + } else { + int32 *buf; + if (data_len <= small_data_array.size() * sizeof(int32)) { + buf = &small_data_array[0]; + } else { + LOG(ERROR) << "Unexpected big unaligned data pointer of length " << slice.size() << " at " << slice.begin(); + data_buf = make_unique<int32[]>(data_len / sizeof(int32)); + buf = data_buf.get(); + } + std::memcpy(static_cast<void *>(buf), static_cast<const void *>(slice.begin()), slice.size()); + data = reinterpret_cast<unsigned char *>(buf); + } + } + + TlParser(const TlParser &other) = delete; + TlParser &operator=(const TlParser &other) = delete; + + void set_error(const string &error_message); + + const char *get_error() const { + if (error.empty()) { + return nullptr; + } + return error.c_str(); + } + + size_t get_error_pos() const { + return error_pos; + } + + Status get_status() const { + if (error.empty()) { + return Status::OK(); + } + return Status::Error(PSLICE() << error << " at " << error_pos); + } + + void check_len(const size_t len) { + if (unlikely(left_len < len)) { + set_error("Not enough data to read"); + } else { + left_len -= len; + } + } + + int32 fetch_int_unsafe() { + int32 result = *reinterpret_cast<const int32 *>(data); + data += sizeof(int32); + return result; + } + + int32 fetch_int() { + check_len(sizeof(int32)); + return fetch_int_unsafe(); + } + + int64 fetch_long_unsafe() { + int64 result; + std::memcpy(reinterpret_cast<unsigned char *>(&result), data, sizeof(int64)); + data += sizeof(int64); + return result; + } + + int64 fetch_long() { + check_len(sizeof(int64)); + return fetch_long_unsafe(); + } + + double fetch_double_unsafe() { + double result; + std::memcpy(reinterpret_cast<unsigned char *>(&result), data, sizeof(double)); + data += sizeof(double); + return result; + } + + double fetch_double() { + check_len(sizeof(double)); + return fetch_double_unsafe(); + } + + template <class T> + T fetch_binary_unsafe() { + T result; + std::memcpy(reinterpret_cast<unsigned char *>(&result), data, sizeof(T)); + data += sizeof(T); + return result; + } + + template <class T> + T fetch_binary() { + static_assert(sizeof(T) <= sizeof(empty_data), "too big fetch_binary"); + static_assert(sizeof(T) % sizeof(int32) == 0, "wrong call to fetch_binary"); + check_len(sizeof(T)); + return fetch_binary_unsafe<T>(); + } + + template <class T> + T fetch_string() { + check_len(sizeof(int32)); + size_t result_len = *data; + const char *result_begin; + size_t result_aligned_len; + if (result_len < 254) { + result_begin = reinterpret_cast<const char *>(data + 1); + result_aligned_len = (result_len >> 2) << 2; + } else if (result_len == 254) { + result_len = data[1] + (data[2] << 8) + (data[3] << 16); + result_begin = reinterpret_cast<const char *>(data + 4); + result_aligned_len = ((result_len + 3) >> 2) << 2; + } else { + set_error("Can't fetch string, 255 found"); + return T(); + } + check_len(result_aligned_len); + data += result_aligned_len + sizeof(int32); + return T(result_begin, result_len); + } + + template <class T> + T fetch_string_raw(const size_t size) { + CHECK(size % sizeof(int32) == 0); + check_len(size); + const char *result = reinterpret_cast<const char *>(data); + data += size; + return T(result, size); + } + + void fetch_end() { + if (left_len) { + set_error("Too much data to fetch"); + } + } + + size_t get_left_len() const { + return left_len; + } +}; + +class TlBufferParser : public TlParser { + public: + explicit TlBufferParser(const BufferSlice *buffer_slice) : TlParser(buffer_slice->as_slice()), parent_(buffer_slice) { + } + template <class T> + T fetch_string() { + auto result = TlParser::fetch_string<T>(); + for (auto &c : result) { + if (c == '\0') { + c = ' '; + } + } + if (check_utf8(result)) { + return result; + } + CHECK(!result.empty()); + LOG(WARNING) << "Wrong UTF-8 string [[" << result << "]] in " << format::as_hex_dump<4>(parent_->as_slice()); + + // trying to remove last character + size_t new_size = result.size() - 1; + while (new_size != 0 && !is_utf8_character_first_code_unit(static_cast<unsigned char>(result[new_size]))) { + new_size--; + } + result.resize(new_size); + if (check_utf8(result)) { + return result; + } + + return T(); + } + template <class T> + T fetch_string_raw(const size_t size) { + return TlParser::fetch_string_raw<T>(size); + } + + private: + const BufferSlice *parent_; + + BufferSlice as_buffer_slice(Slice slice) { + if (is_aligned_pointer<4>(slice.data())) { + return parent_->from_slice(slice); + } + return BufferSlice(slice); + } +}; + +template <> +inline BufferSlice TlBufferParser::fetch_string<BufferSlice>() { + return as_buffer_slice(TlParser::fetch_string<Slice>()); +} + +template <> +inline BufferSlice TlBufferParser::fetch_string_raw<BufferSlice>(const size_t size) { + return as_buffer_slice(TlParser::fetch_string_raw<Slice>(size)); +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/tl_storers.h b/libs/tdlib/td/tdutils/td/utils/tl_storers.h new file mode 100644 index 0000000000..f389451d8a --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/tl_storers.h @@ -0,0 +1,281 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/int_types.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Slice.h" +#include "td/utils/StorerBase.h" + +#include <cstring> + +namespace td { + +class TlStorerUnsafe { + char *buf; + + public: + explicit TlStorerUnsafe(char *buf) : buf(buf) { + CHECK(is_aligned_pointer<4>(buf)); + } + + TlStorerUnsafe(const TlStorerUnsafe &other) = delete; + TlStorerUnsafe &operator=(const TlStorerUnsafe &other) = delete; + + template <class T> + void store_binary(const T &x) { + std::memcpy(buf, reinterpret_cast<const unsigned char *>(&x), sizeof(T)); + buf += sizeof(T); + } + + void store_int(int32 x) { + *reinterpret_cast<int32 *>(buf) = x; + buf += sizeof(int32); + } + + void store_long(int64 x) { + store_binary<int64>(x); + } + + void store_slice(Slice slice) { + std::memcpy(buf, slice.begin(), slice.size()); + buf += slice.size(); + } + void store_storer(const Storer &storer) { + size_t size = storer.store(reinterpret_cast<unsigned char *>(buf)); + buf += size; + } + + template <class T> + void store_string(const T &str) { + size_t len = str.size(); + if (len < 254) { + *buf++ = static_cast<char>(len); + len++; + } else if (len < (1 << 24)) { + *buf++ = static_cast<char>(static_cast<unsigned char>(254)); + *buf++ = static_cast<char>(len & 255); + *buf++ = static_cast<char>((len >> 8) & 255); + *buf++ = static_cast<char>(len >> 16); + } else { + LOG(FATAL) << "String size " << len << " is too big to be stored"; + } + std::memcpy(buf, str.data(), str.size()); + buf += str.size(); + + switch (len & 3) { + case 1: + *buf++ = '\0'; + // fallthrough + case 2: + *buf++ = '\0'; + // fallthrough + case 3: + *buf++ = '\0'; + } + } + + char *get_buf() const { + return buf; + } +}; + +class TlStorerCalcLength { + size_t length = 0; + + public: + TlStorerCalcLength() = default; + TlStorerCalcLength(const TlStorerCalcLength &other) = delete; + TlStorerCalcLength &operator=(const TlStorerCalcLength &other) = delete; + + template <class T> + void store_binary(const T &x) { + length += sizeof(T); + } + + void store_int(int32 x) { + store_binary<int32>(x); + } + + void store_long(int64 x) { + store_binary<int64>(x); + } + + void store_slice(Slice slice) { + length += slice.size(); + } + + void store_storer(const Storer &storer) { + length += storer.size(); + } + + template <class T> + void store_string(const T &str) { + size_t add = str.size(); + if (add < 254) { + add += 1; + } else { + add += 4; + } + add = (add + 3) & -4; + length += add; + } + + size_t get_length() const { + return length; + } +}; + +class TlStorerToString { + std::string result; + int shift = 0; + + void store_field_begin(const char *name) { + for (int i = 0; i < shift; i++) { + result += ' '; + } + if (name && name[0]) { + result += name; + result += " = "; + } + } + + void store_field_end() { + result += "\n"; + } + + void store_long(int64 value) { + result += (PSLICE() << value).c_str(); + } + + void store_binary(Slice data) { + static const char *hex = "0123456789ABCDEF"; + + result.append("{ "); + for (auto c : data) { + unsigned char byte = c; + result += hex[byte >> 4]; + result += hex[byte & 15]; + result += ' '; + } + result.append("}"); + } + + public: + TlStorerToString() = default; + TlStorerToString(const TlStorerToString &other) = delete; + TlStorerToString &operator=(const TlStorerToString &other) = delete; + + void store_field(const char *name, bool value) { + store_field_begin(name); + result += (value ? "true" : "false"); + store_field_end(); + } + + void store_field(const char *name, int32 value) { + store_field(name, static_cast<int64>(value)); + } + + void store_field(const char *name, int64 value) { + store_field_begin(name); + store_long(value); + store_field_end(); + } + + void store_field(const char *name, double value) { + store_field_begin(name); + result += (PSLICE() << value).c_str(); + store_field_end(); + } + + void store_field(const char *name, const char *value) { + store_field_begin(name); + result += value; + store_field_end(); + } + + void store_field(const char *name, const string &value) { + store_field_begin(name); + result += '"'; + result.append(value.data(), value.size()); + result += '"'; + store_field_end(); + } + + template <class T> + void store_field(const char *name, const T &value) { + store_field_begin(name); + result.append(value.data(), value.size()); + store_field_end(); + } + + template <class BytesT> + void store_bytes_field(const char *name, const BytesT &value) { + static const char *hex = "0123456789ABCDEF"; + + store_field_begin(name); + result.append("bytes { "); + for (size_t i = 0; i < value.size(); i++) { + int b = value[static_cast<int>(i)] & 0xff; + result += hex[b >> 4]; + result += hex[b & 15]; + result += ' '; + } + result.append("}"); + store_field_end(); + } + + void store_field(const char *name, const UInt128 &value) { + store_field_begin(name); + store_binary(Slice(reinterpret_cast<const unsigned char *>(&value), sizeof(value))); + store_field_end(); + } + + void store_field(const char *name, const UInt256 &value) { + store_field_begin(name); + store_binary(Slice(reinterpret_cast<const unsigned char *>(&value), sizeof(value))); + store_field_end(); + } + + void store_class_begin(const char *field_name, const char *class_name) { + store_field_begin(field_name); + result += class_name; + result += " {\n"; + shift += 2; + } + + void store_class_end() { + shift -= 2; + for (int i = 0; i < shift; i++) { + result += ' '; + } + result += "}\n"; + CHECK(shift >= 0); + } + + std::string str() const { + return result; + } +}; + +template <class T> +size_t tl_calc_length(const T &data) { + TlStorerCalcLength storer_calc_length; + data.store(storer_calc_length); + return storer_calc_length.get_length(); +} + +template <class T, class CharT> +size_t tl_store_unsafe(const T &data, CharT *dst) { + char *start = reinterpret_cast<char *>(dst); + TlStorerUnsafe storer_unsafe(start); + data.store(storer_unsafe); + return storer_unsafe.get_buf() - start; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/type_traits.h b/libs/tdlib/td/tdutils/td/utils/type_traits.h new file mode 100644 index 0000000000..ef9c159420 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/type_traits.h @@ -0,0 +1,22 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +namespace td { + +template <class FunctionT> +struct member_function_class; + +template <class ReturnType, class Type> +struct member_function_class<ReturnType Type::*> { + using type = Type; +}; + +template <class FunctionT> +using member_function_class_t = typename member_function_class<FunctionT>::type; + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/unicode.cpp b/libs/tdlib/td/tdutils/td/utils/unicode.cpp new file mode 100644 index 0000000000..11e76b7979 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/unicode.cpp @@ -0,0 +1,574 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/unicode.h" + +#include "td/utils/logging.h" + +#include <algorithm> +#include <iterator> + +namespace td { + +// list of [(range_begin << 5) + range_type] +static const uint32 unicode_simple_category_ranges[] = { + 0, 1028, 1056, 1538, 1856, 2081, 2912, 3105, 3936, 5124, 5152, 5441, + 5472, 5699, 5760, 5793, 5824, 5923, 5953, 5984, 6019, 6112, 6145, 6880, + 6913, 7904, 7937, 22592, 22721, 23104, 23553, 23712, 23937, 23968, 24001, 24032, + 28161, 28320, 28353, 28416, 28481, 28608, 28641, 28672, 28865, 28896, 28929, 29024, + 29057, 29088, 29121, 29760, 29793, 32448, 32481, 36928, 37185, 42496, 42529, 43744, + 43809, 43840, 44065, 45312, 47617, 48480, 48641, 48736, 50177, 51552, 52226, 52544, + 52673, 52736, 52769, 55936, 55969, 56000, 56481, 56544, 56769, 56834, 57153, 57248, + 57313, 57344, 57857, 57888, 57921, 58880, 59809, 62656, 63009, 63040, 63490, 63809, + 64864, 65153, 65216, 65345, 65376, 65537, 66240, 66369, 66400, 66689, 66720, 66817, + 66848, 67585, 68384, 70657, 71328, 71361, 71616, 73857, 75584, 75681, 75712, 76289, + 76320, 76545, 76864, 76994, 77312, 77345, 77856, 77985, 78240, 78305, 78368, 78433, + 79136, 79169, 79392, 79425, 79456, 79553, 79680, 79777, 79808, 80321, 80352, 80769, + 80832, 80865, 80960, 81090, 81409, 81472, 81539, 81728, 82081, 82272, 82401, 82464, + 82529, 83232, 83265, 83488, 83521, 83584, 83617, 83680, 83713, 83776, 84769, 84896, + 84929, 84960, 85186, 85504, 85569, 85664, 86177, 86464, 86497, 86592, 86625, 87328, + 87361, 87584, 87617, 87680, 87713, 87872, 87969, 88000, 88577, 88608, 89089, 89152, + 89282, 89600, 89889, 89920, 90273, 90528, 90593, 90656, 90721, 91424, 91457, 91680, + 91713, 91776, 91809, 91968, 92065, 92096, 93057, 93120, 93153, 93248, 93378, 93696, + 93729, 93763, 93952, 94305, 94336, 94369, 94560, 94657, 94752, 94785, 94912, 95009, + 95072, 95105, 95136, 95169, 95232, 95329, 95392, 95489, 95584, 95681, 96064, 96769, + 96800, 97474, 97795, 97888, 98465, 98720, 98753, 98848, 98881, 99616, 99649, 100160, + 100257, 100288, 101121, 101216, 101377, 101440, 101570, 101888, 102147, 102368, 102401, 102432, + 102561, 102816, 102849, 102944, 102977, 103712, 103745, 104064, 104097, 104256, 104353, 104384, + 105409, 105440, 105473, 105536, 105666, 105984, 106017, 106080, 106657, 106912, 106945, 107040, + 107073, 108384, 108449, 108480, 108993, 109024, 109185, 109280, 109315, 109537, 109632, 109762, + 110083, 110368, 110401, 110592, 110753, 111328, 111425, 112192, 112225, 112512, 112545, 112576, + 112641, 112864, 113858, 114176, 114721, 116256, 116289, 116352, 116737, 116960, 117250, 117568, + 118817, 118880, 118913, 118944, 119009, 119072, 119105, 119136, 119201, 119232, 119425, 119552, + 119585, 119808, 119841, 119936, 119969, 120000, 120033, 120064, 120129, 120192, 120225, 120352, + 120385, 120448, 120737, 120768, 120833, 120992, 121025, 121056, 121346, 121664, 121729, 121856, + 122881, 122912, 123906, 124227, 124544, 124929, 125184, 125217, 126368, 127233, 127392, 131073, + 132448, 133089, 133122, 133440, 133633, 133824, 133953, 134080, 134177, 134208, 134305, 134368, + 134593, 134688, 134817, 135232, 135617, 135648, 135682, 136000, 136193, 137408, 137441, 137472, + 137633, 137664, 137729, 139104, 139137, 149792, 149825, 149952, 150017, 150240, 150273, 150304, + 150337, 150464, 150529, 151840, 151873, 152000, 152065, 153120, 153153, 153280, 153345, 153568, + 153601, 153632, 153665, 153792, 153857, 154336, 154369, 156192, 156225, 156352, 156417, 158560, + 159011, 159648, 159745, 160256, 160769, 163520, 163585, 163776, 163873, 183712, 183777, 184324, + 184353, 185184, 185345, 187744, 187843, 187937, 188192, 188417, 188832, 188865, 188992, 189441, + 190016, 190465, 191040, 191489, 191904, 191937, 192032, 192513, 194176, 195297, 195328, 195457, + 195488, 195586, 195904, 196099, 196416, 197122, 197440, 197633, 200448, 200705, 200864, 200929, + 202016, 202049, 202080, 202241, 204480, 204801, 205792, 207042, 207361, 208320, 208385, 208544, + 208897, 210304, 210433, 211264, 211458, 211779, 211808, 212993, 213728, 214017, 215712, 217090, + 217408, 217602, 217920, 218337, 218368, 221345, 222848, 223393, 223616, 223746, 224064, 225377, + 226336, 226753, 226818, 227137, 228544, 229377, 230528, 231426, 231744, 231841, 231938, 232257, + 233408, 233473, 233760, 236833, 236960, 236993, 237120, 237217, 237280, 237569, 243712, 245761, + 254656, 254721, 254912, 254977, 256192, 256257, 256448, 256513, 256768, 256801, 256832, 256865, + 256896, 256929, 256960, 256993, 257984, 258049, 259744, 259777, 260000, 260033, 260064, 260161, + 260256, 260289, 260512, 260609, 260736, 260801, 260992, 261121, 261536, 261697, 261792, 261825, + 262048, 262148, 262496, 263428, 263488, 263652, 263680, 265188, 265216, 265731, 265761, 265792, + 265859, 266048, 266209, 266243, 266560, 266753, 267168, 270401, 270432, 270561, 270592, 270657, + 270976, 271009, 271040, 271137, 271296, 271489, 271520, 271553, 271584, 271617, 271648, 271681, + 271808, 271841, 272192, 272257, 272384, 272545, 272704, 272833, 272864, 272899, 274529, 274595, + 274752, 297987, 299904, 302403, 303104, 323267, 324224, 360449, 361952, 361985, 363488, 363521, + 367776, 367969, 368096, 368193, 368256, 368547, 368576, 368641, 369856, 369889, 369920, 370081, + 370112, 370177, 371968, 372193, 372224, 372737, 373472, 373761, 373984, 374017, 374240, 374273, + 374496, 374529, 374752, 374785, 375008, 375041, 375264, 375297, 375520, 375553, 375776, 378337, + 378368, 393220, 393248, 393377, 393443, 393472, 394275, 394560, 394785, 394944, 395011, 395105, + 395168, 395297, 398048, 398241, 398336, 398369, 401248, 401281, 401408, 401569, 402880, 402977, + 405984, 406083, 406208, 406529, 407392, 409089, 409600, 410627, 410944, 411907, 412160, 412195, + 412672, 413699, 414016, 415267, 415744, 425985, 636608, 638977, 1309376, 1310721, 1348000, 1350145, + 1351616, 1351681, 1360288, 1360385, 1360898, 1361217, 1361280, 1361921, 1363424, 1363937, 1364928, 1364993, + 1367235, 1367552, 1368801, 1369088, 1369153, 1372448, 1372513, 1373664, 1373697, 1373952, 1375969, 1376320, + 1376353, 1376448, 1376481, 1376608, 1376641, 1377376, 1377795, 1377984, 1378305, 1379968, 1380417, 1382016, + 1382914, 1383232, 1384001, 1384192, 1384289, 1384320, 1384353, 1384384, 1384450, 1384769, 1385664, 1385985, + 1386720, 1387521, 1388448, 1388673, 1390176, 1391073, 1391106, 1391424, 1391617, 1391776, 1391809, 1392130, + 1392449, 1392608, 1392641, 1393952, 1394689, 1394784, 1394817, 1395072, 1395202, 1395520, 1395713, 1396448, + 1396545, 1396576, 1396673, 1398272, 1398305, 1398336, 1398433, 1398496, 1398561, 1398720, 1398785, 1398816, + 1398849, 1398880, 1399649, 1399744, 1399809, 1400160, 1400385, 1400480, 1400865, 1401056, 1401121, 1401312, + 1401377, 1401568, 1401857, 1402080, 1402113, 1402336, 1402369, 1403744, 1403777, 1404096, 1404417, 1408096, + 1408514, 1408832, 1409025, 1766528, 1766913, 1767648, 1767777, 1769344, 2039809, 2051520, 2051585, 2054976, + 2056193, 2056416, 2056801, 2056960, 2057121, 2057152, 2057185, 2057504, 2057537, 2057952, 2057985, 2058144, + 2058177, 2058208, 2058241, 2058304, 2058337, 2058400, 2058433, 2061888, 2062945, 2074560, 2075137, 2077184, + 2077249, 2078976, 2080257, 2080640, 2084353, 2084512, 2084545, 2088864, 2089474, 2089792, 2090017, 2090848, + 2091041, 2091872, 2092225, 2095072, 2095169, 2095360, 2095425, 2095616, 2095681, 2095872, 2095937, 2096032, + 2097153, 2097536, 2097569, 2098400, 2098433, 2099040, 2099073, 2099136, 2099169, 2099648, 2099713, 2100160, + 2101249, 2105184, 2105571, 2107008, 2107395, 2109216, 2109763, 2109824, 2117633, 2118560, 2118657, 2120224, + 2120739, 2121600, 2121729, 2122755, 2122880, 2123265, 2123811, 2123841, 2124099, 2124128, 2124289, 2125504, + 2125825, 2126784, 2126849, 2128000, 2128129, 2128384, 2128419, 2128576, 2129921, 2134976, 2135042, 2135360, + 2135553, 2136704, 2136833, 2137984, 2138113, 2139392, 2139649, 2141312, 2146305, 2156256, 2156545, 2157248, + 2157569, 2157824, 2162689, 2162880, 2162945, 2162976, 2163009, 2164416, 2164449, 2164512, 2164609, 2164640, + 2164705, 2165440, 2165507, 2165761, 2166496, 2166563, 2166785, 2167776, 2168035, 2168320, 2169857, 2170464, + 2170497, 2170560, 2170723, 2170881, 2171587, 2171776, 2171905, 2172736, 2174977, 2176768, 2176899, 2176961, + 2177027, 2177536, 2177603, 2179073, 2179104, 2179585, 2179712, 2179745, 2179840, 2179873, 2180736, 2181123, + 2181376, 2182145, 2183075, 2183136, 2183169, 2184099, 2184192, 2185217, 2185472, 2185505, 2186400, 2186595, + 2186752, 2187265, 2188992, 2189313, 2190016, 2190083, 2190337, 2190944, 2191107, 2191361, 2191936, 2192675, + 2192896, 2195457, 2197792, 2199553, 2201184, 2201601, 2203232, 2203459, 2203648, 2214915, 2215904, 2228321, + 2230016, 2230851, 2231490, 2231808, 2232417, 2233856, 2234881, 2235680, 2235906, 2236224, 2236513, 2237664, + 2238146, 2238464, 2238977, 2240096, 2240193, 2240224, 2240609, 2242144, 2242593, 2242720, 2243074, 2243393, + 2243424, 2243457, 2243488, 2243619, 2244256, 2244609, 2245184, 2245217, 2246016, 2248705, 2248928, 2248961, + 2248992, 2249025, 2249152, 2249185, 2249664, 2249697, 2250016, 2250241, 2251744, 2252290, 2252608, 2252961, + 2253216, 2253281, 2253344, 2253409, 2254112, 2254145, 2254368, 2254401, 2254464, 2254497, 2254656, 2254753, + 2254784, 2255361, 2255392, 2255777, 2255936, 2260993, 2262688, 2263265, 2263392, 2263554, 2263872, 2265089, + 2266624, 2267265, 2267328, 2267361, 2267392, 2267650, 2267968, 2273281, 2274784, 2276097, 2276224, 2277377, + 2278912, 2279553, 2279584, 2279938, 2280256, 2281473, 2282848, 2283522, 2283840, 2285569, 2286400, 2287106, + 2287427, 2287488, 2298881, 2300930, 2301251, 2301536, 2301921, 2301952, 2316289, 2318112, 2326529, 2326816, + 2326849, 2328032, 2328577, 2328608, 2329090, 2329411, 2330016, 2330177, 2331136, 2359297, 2388800, 2392067, + 2395616, 2396161, 2402432, 2490369, 2524640, 2654209, 2672864, 2949121, 2967328, 2967553, 2968544, 2968578, + 2968896, 2972161, 2973120, 2973697, 2975232, 2975745, 2975872, 2976258, 2976576, 2976611, 2976832, 2976865, + 2977536, 2977697, 2978304, 3006465, 3008672, 3009025, 3009056, 3011169, 3011584, 3013633, 3013664, 3014657, + 3210656, 3211265, 3235424, 3538945, 3539008, 3637249, 3640672, 3640833, 3641248, 3641345, 3641632, 3641857, + 3642176, 3828739, 3829312, 3833857, 3836576, 3836609, 3838880, 3838913, 3838976, 3839041, 3839072, 3839137, + 3839200, 3839265, 3839392, 3839425, 3839808, 3839841, 3839872, 3839905, 3840128, 3840161, 3842240, 3842273, + 3842400, 3842465, 3842720, 3842753, 3842976, 3843009, 3843904, 3843937, 3844064, 3844097, 3844256, 3844289, + 3844320, 3844417, 3844640, 3844673, 3855552, 3855617, 3856416, 3856449, 3857248, 3857281, 3858272, 3858305, + 3859104, 3859137, 3860128, 3860161, 3860960, 3860993, 3861984, 3862017, 3862816, 3862849, 3863840, 3863873, + 3864672, 3864705, 3864960, 3865026, 3866624, 3997697, 4004000, 4004067, 4004352, 4005889, 4008064, 4008450, + 4008768, 4046849, 4046976, 4047009, 4047872, 4047905, 4047968, 4048001, 4048032, 4048097, 4048128, 4048161, + 4048480, 4048513, 4048640, 4048673, 4048704, 4048737, 4048768, 4048961, 4048992, 4049121, 4049152, 4049185, + 4049216, 4049249, 4049280, 4049313, 4049408, 4049441, 4049504, 4049537, 4049568, 4049633, 4049664, 4049697, + 4049728, 4049761, 4049792, 4049825, 4049856, 4049889, 4049920, 4049953, 4050016, 4050049, 4050080, 4050145, + 4050272, 4050305, 4050528, 4050561, 4050688, 4050721, 4050848, 4050881, 4050912, 4050945, 4051264, 4051297, + 4051840, 4052001, 4052096, 4052129, 4052288, 4052321, 4052864, 4071427, 4071840, 4194305, 5561056, 5562369, + 5695136, 5695489, 5702592, 5702657, 5887040, 6225921, 6243264, 4294967295}; + +static constexpr uint32 TABLE_SIZE = 1280; + +static int16 prepare_search_character_table[TABLE_SIZE] = { + 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 32, 32, 32, 32, 32, 32, 32, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 32, 32, 32, 32, + 32, 32, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 170, + 32, 32, 0, 32, 32, 32, 32, 178, 179, 32, 956, 32, 0, 32, 185, 186, 32, 188, 189, + 190, 32, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 32, 248, 249, 250, 251, 252, 253, 254, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 32, 248, 249, 250, 251, 252, 253, 254, 255, 257, 257, 259, 259, 261, 261, 263, 263, 265, 265, + 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277, 279, 279, 281, 281, 283, 283, 285, + 285, 287, 287, 289, 289, 291, 291, 293, 293, 295, 295, 297, 297, 299, 299, 301, 301, 303, 303, + 105, 305, 307, 307, 309, 309, 311, 311, 312, 314, 314, 316, 316, 318, 318, 320, 320, 322, 322, + 324, 324, 326, 326, 328, 328, 329, 331, 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341, + 343, 343, 345, 345, 347, 347, 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361, + 361, 363, 363, 365, 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 255, 378, 378, 380, + 380, 382, 382, 115, 384, 595, 387, 387, 389, 389, 596, 392, 392, 598, 599, 396, 396, 397, 477, + 601, 603, 402, 402, 608, 611, 405, 617, 616, 409, 409, 410, 411, 623, 626, 414, 629, 417, 417, + 419, 419, 421, 421, 640, 424, 424, 643, 426, 427, 429, 429, 648, 432, 432, 650, 651, 436, 436, + 438, 438, 658, 441, 441, 442, 443, 445, 445, 446, 447, 448, 449, 450, 451, 454, 454, 454, 457, + 457, 457, 460, 460, 460, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470, 472, 472, 474, 474, + 476, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, 487, 487, 489, 489, 491, 491, 493, 493, + 495, 495, 496, 499, 499, 499, 501, 501, 405, 447, 505, 505, 507, 507, 509, 509, 511, 511, 513, + 513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, 525, 525, 527, 527, 529, 529, 531, 531, + 533, 533, 535, 535, 537, 537, 539, 539, 541, 541, 543, 543, 414, 545, 547, 547, 549, 549, 551, + 551, 553, 553, 555, 555, 557, 557, 559, 559, 561, 561, 563, 563, 564, 565, 566, 567, 568, 569, + 11365, 572, 572, 410, 11366, 575, 576, 578, 578, 384, 649, 652, 583, 583, 585, 585, 587, 587, 589, + 589, 591, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, + 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, + 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, + 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, + 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 0, 701, 0, + 703, 704, 705, 32, 32, 32, 32, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 736, 737, 738, 739, 740, + 32, 32, 32, 32, 32, 32, 32, 748, 32, 750, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 881, 881, 883, 883, 884, 32, 887, 887, 888, 889, 890, 891, 892, + 893, 32, 1011, 896, 897, 898, 899, 32, 32, 940, 32, 941, 942, 943, 907, 972, 909, 973, 974, + 912, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 930, + 963, 964, 965, 966, 967, 968, 969, 970, 971, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, + 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 963, 963, 964, 965, 966, 967, 968, + 969, 970, 971, 972, 973, 974, 983, 946, 952, 965, 965, 965, 966, 960, 983, 985, 985, 987, 987, + 989, 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005, 1005, 1007, + 1007, 954, 961, 963, 1011, 952, 949, 32, 1016, 1016, 1010, 1019, 1019, 1020, 891, 892, 893, 1104, 1105, + 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1072, 1073, 1074, 1075, 1076, + 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, + 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, + 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, + 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, + 1121, 1123, 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137, 1137, 1139, 1139, + 1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151, 1151, 1153, 1153, 32, 0, 0, 0, 0, + 0, 0, 0, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173, 1173, 1175, 1175, 1177, 1177, + 1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187, 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, + 1197, 1199, 1199, 1201, 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215, 1215, + 1231, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230, 1230, 1231, 1233, 1233, 1235, + 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245, 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253, + 1255, 1255, 1257, 1257, 1259, 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273, + 1273, 1275, 1275, 1277, 1277, 1279, 1279}; + +static const int32 prepare_search_character_ranges[] = { + 1280, 2097153, 1328, 1328, 1329, -1378, 1367, -1368, 1370, 32, 1376, -1377, + 1417, 32, 1419, -1420, 1421, 32, 1424, 1424, 1425, 0, 1470, 32, + 1471, 0, 1472, 32, 1473, 0, 1475, 32, 1476, 0, 1478, 32, + 1479, 0, 1480, -1481, 1523, 32, 1525, -1526, 1536, 0, 1542, 32, + 1552, 0, 1563, 32, 1564, 0, 1565, 1565, 1566, 32, 1568, -1569, + 1611, 0, 1632, -1633, 1642, 32, 1646, -1647, 1648, 0, 1649, -1650, + 1748, 32, 1749, 1749, 1750, 0, 1758, 32, 1759, 0, 1765, -1766, + 1767, 0, 1769, 32, 1770, 0, 1774, -1775, 1789, 32, 1791, 1791, + 1792, 32, 1806, 1806, 1807, 0, 1808, 1808, 1809, 0, 1810, -1811, + 1840, 0, 1867, -1868, 1958, 0, 1969, -1970, 2027, 0, 2036, -2037, + 2038, 32, 2042, -2043, 2070, 0, 2074, 2074, 2075, 0, 2084, 2084, + 2085, 0, 2088, 2088, 2089, 0, 2094, -2095, 2096, 32, 2111, -2112, + 2137, 0, 2140, -2141, 2142, 32, 2143, -2144, 2260, 0, 2308, -2309, + 2362, 0, 2365, 2365, 2366, 0, 2384, 2384, 2385, 0, 2392, -2393, + 2402, 0, 2404, 32, 2406, -2407, 2416, 32, 2417, -2418, 2433, 0, + 2436, -2437, 2492, 0, 2493, 2493, 2494, 0, 2501, -2502, 2503, 0, + 2505, -2506, 2507, 0, 2510, -2511, 2519, 0, 2520, -2521, 2530, 0, + 2532, -2533, 2546, 32, 2548, -2549, 2554, 32, 2556, -2557, 2561, 0, + 2564, -2565, 2620, 0, 2621, 2621, 2622, 0, 2627, -2628, 2631, 0, + 2633, -2634, 2635, 0, 2638, -2639, 2641, 0, 2642, -2643, 2672, 0, + 2674, -2675, 2677, 0, 2678, -2679, 2689, 0, 2692, -2693, 2748, 0, + 2749, 2749, 2750, 0, 2758, 2758, 2759, 0, 2762, 2762, 2763, 0, + 2766, -2767, 2786, 0, 2788, -2789, 2800, 32, 2802, -2803, 2817, 0, + 2820, -2821, 2876, 0, 2877, 2877, 2878, 0, 2885, -2886, 2887, 0, + 2889, -2890, 2891, 0, 2894, -2895, 2902, 0, 2904, -2905, 2914, 0, + 2916, -2917, 2928, 32, 2929, -2930, 2946, 0, 2947, -2948, 3006, 0, + 3011, -3012, 3014, 0, 3017, 3017, 3018, 0, 3022, -3023, 3031, 0, + 3032, -3033, 3059, 32, 3067, -3068, 3072, 0, 3076, -3077, 3134, 0, + 3141, 3141, 3142, 0, 3145, 3145, 3146, 0, 3150, -3151, 3157, 0, + 3159, -3160, 3170, 0, 3172, -3173, 3199, 32, 3200, 3200, 3201, 0, + 3204, -3205, 3260, 0, 3261, 3261, 3262, 0, 3269, 3269, 3270, 0, + 3273, 3273, 3274, 0, 3278, -3279, 3285, 0, 3287, -3288, 3298, 0, + 3300, -3301, 3329, 0, 3332, -3333, 3390, 0, 3397, 3397, 3398, 0, + 3401, 3401, 3402, 0, 3406, 3406, 3407, 32, 3408, -3409, 3415, 0, + 3416, -3417, 3426, 0, 3428, -3429, 3449, 32, 3450, -3451, 3458, 0, + 3460, -3461, 3530, 0, 3531, -3532, 3535, 0, 3541, 3541, 3542, 0, + 3543, 3543, 3544, 0, 3552, -3553, 3570, 0, 3572, 32, 3573, -3574, + 3633, 0, 3634, -3635, 3636, 0, 3643, -3644, 3647, 32, 3648, -3649, + 3655, 0, 3663, 32, 3664, -3665, 3674, 32, 3676, -3677, 3761, 0, + 3762, -3763, 3764, 0, 3770, 3770, 3771, 0, 3773, -3774, 3784, 0, + 3790, -3791, 3841, 32, 3864, 0, 3866, 32, 3872, -3873, 3892, 32, + 3893, 0, 3894, 32, 3895, 0, 3896, 32, 3897, 0, 3898, 32, + 3902, 0, 3904, -3905, 3953, 0, 3973, 32, 3974, 0, 3976, -3977, + 3981, 0, 3992, 3992, 3993, 0, 4029, 4029, 4030, 32, 4038, 0, + 4039, 32, 4045, 4045, 4046, 32, 4059, -4060, 4139, 0, 4159, -4160, + 4170, 32, 4176, -4177, 4182, 0, 4186, -4187, 4190, 0, 4193, 4193, + 4194, 0, 4197, -4198, 4199, 0, 4206, -4207, 4209, 0, 4213, -4214, + 4226, 0, 4238, 4238, 4239, 0, 4240, -4241, 4250, 0, 4254, 32, + 4256, -11521, 4294, 4294, 4295, 11559, 4296, -4297, 4301, 11565, 4302, -4303, + 4347, 32, 4348, -4349, 4957, 0, 4960, 32, 4969, -4970, 5008, 32, + 5018, -5019, 5112, -5105, 5118, -5119, 5120, 32, 5121, -5122, 5741, 32, + 5743, -5744, 5760, 32, 5761, -5762, 5787, 32, 5789, -5790, 5867, 32, + 5870, -5871, 5906, 0, 5909, -5910, 5938, 0, 5941, 32, 5943, -5944, + 5970, 0, 5972, -5973, 6002, 0, 6004, -6005, 6068, 0, 6100, 32, + 6103, 6103, 6104, 32, 6108, 6108, 6109, 0, 6110, -6111, 6144, 32, + 6155, 0, 6159, -6160, 6277, 0, 6279, -6280, 6313, 0, 6314, -6315, + 6432, 0, 6444, -6445, 6448, 0, 6460, -6461, 6464, 32, 6465, -6466, + 6468, 32, 6470, -6471, 6622, 32, 6656, -6657, 6679, 0, 6684, -6685, + 6686, 32, 6688, -6689, 6741, 0, 6751, 6751, 6752, 0, 6781, -6782, + 6783, 0, 6784, -6785, 6816, 32, 6823, 6823, 6824, 32, 6830, -6831, + 6832, 0, 6847, -6848, 6912, 0, 6917, -6918, 6964, 0, 6981, -6982, + 7002, 32, 7019, 0, 7028, 32, 7037, -7038, 7040, 0, 7043, -7044, + 7073, 0, 7086, -7087, 7142, 0, 7156, -7157, 7164, 32, 7168, -7169, + 7204, 0, 7224, -7225, 7227, 32, 7232, -7233, 7294, 32, 7296, 1074, + 7297, 1076, 7298, 1086, 7299, -1090, 7301, 1090, 7302, 1098, 7303, 1123, + 7304, 42571, 7305, -7306, 7360, 32, 7368, -7369, 7376, 0, 7379, 32, + 7380, 0, 7401, -7402, 7405, 0, 7406, -7407, 7410, 0, 7413, -7414, + 7416, 0, 7418, -7419, 7468, 97, 7469, 230, 7470, 98, 7471, 7471, + 7472, -101, 7474, 477, 7475, -104, 7483, 7483, 7484, 111, 7485, 547, + 7486, 112, 7487, 114, 7488, -117, 7490, 119, 7491, -7492, 7616, 0, + 7670, -7671, 7675, 0, 7680, 2097153, 7830, -7831, 7835, 7777, 7836, -7837, + 7838, 223, 7839, 2097153, 7936, -7937, 7944, -7937, 7952, -7953, 7960, -7953, + 7966, -7967, 7976, -7969, 7984, -7985, 7992, -7985, 8000, -8001, 8008, -8001, + 8014, -8015, 8025, 8017, 8026, 8026, 8027, 8019, 8028, 8028, 8029, 8021, + 8030, 8030, 8031, 8023, 8032, -8033, 8040, -8033, 8048, -8049, 8072, -8065, + 8080, -8081, 8088, -8081, 8096, -8097, 8104, -8097, 8112, -8113, 8120, -8113, + 8122, -8049, 8124, 8115, 8125, 32, 8126, 953, 8127, 32, 8130, -8131, + 8136, -8051, 8140, 8131, 8141, 32, 8144, -8145, 8152, -8145, 8154, -8055, + 8156, 8156, 8157, 32, 8160, -8161, 8168, -8161, 8170, -8059, 8172, 8165, + 8173, 32, 8176, -8177, 8184, -8057, 8186, -8061, 8188, 8179, 8189, 32, + 8191, 8191, 8192, 32, 8203, 0, 8208, 32, 8234, 0, 8239, 32, + 8288, 0, 8293, 8293, 8294, 0, 8304, -8305, 8314, 32, 8319, -8320, + 8330, 32, 8335, -8336, 8352, 32, 8383, -8384, 8400, 0, 8433, -8434, + 8448, 32, 8450, 99, 8452, 32, 8455, 603, 8456, 32, 8457, 102, + 8458, 8458, 8459, 104, 8462, -8463, 8464, 105, 8466, 108, 8467, 8467, + 8468, 32, 8469, 110, 8470, 32, 8473, -113, 8476, 114, 8478, 32, + 8484, 122, 8485, 32, 8486, 969, 8487, 32, 8488, 122, 8489, 32, + 8490, 107, 8491, 229, 8492, -99, 8494, 32, 8495, 8495, 8496, -102, + 8498, 8526, 8499, 109, 8500, -8501, 8506, 32, 8508, -8509, 8510, 947, + 8511, 960, 8512, 32, 8517, 100, 8518, -8519, 8522, 32, 8526, 8526, + 8527, 32, 8528, -8529, 8544, -8561, 8560, -8561, 8579, 8580, 8581, -8582, + 8586, 32, 8588, -8589, 8592, 32, 9215, 9215, 9216, 32, 9255, -9256, + 9280, 32, 9291, -9292, 9372, 32, 9398, -9425, 9424, -9425, 9472, 32, + 10102, -10103, 10132, 32, 11124, -11125, 11126, 32, 11158, -11159, 11160, 32, + 11194, -11195, 11197, 32, 11209, 11209, 11210, 32, 11218, -11219, 11244, 32, + 11248, -11249, 11264, -11313, 11311, -11312, 11360, 11361, 11362, 619, 11363, 7549, + 11364, 637, 11365, -11366, 11367, 11368, 11369, 11370, 11371, 11372, 11373, 593, + 11374, 625, 11375, 592, 11376, 594, 11377, 2097153, 11380, 11380, 11381, 11382, + 11383, -11384, 11389, 118, 11390, -576, 11392, 2097153, 11492, 11492, 11493, 32, + 11499, 11500, 11501, 11502, 11503, 0, 11506, 11507, 11508, -11509, 11513, 32, + 11517, 11517, 11518, 32, 11520, -11521, 11632, 32, 11633, -11634, 11647, 0, + 11648, -11649, 11744, 0, 11776, 32, 11823, 11823, 11824, 32, 11845, -11846, + 11904, 32, 11930, 11930, 11931, 32, 11935, 11935, 11936, 32, 12019, -12020, + 12272, 32, 12284, -12285, 12288, 32, 12293, -12294, 12296, 32, 12321, -12322, + 12330, 0, 12336, 32, 12337, -12338, 12342, 32, 12344, -12345, 12349, 32, + 12352, -12353, 12441, 0, 12443, 32, 12445, -12446, 12448, 32, 12449, -12450, + 12539, 32, 12540, 0, 12541, -12542, 12688, 32, 12690, -12691, 12736, 32, + 12772, -12773, 12800, 32, 12831, -12832, 12842, 32, 12868, -12869, 12880, 32, + 12881, -12882, 12910, 32, 12928, -12929, 12992, 32, 13008, -13009, 13056, 32, + 13312, -13313, 19904, 32, 19968, -19969, 42128, 32, 42183, -42184, 42238, 32, + 42240, -42241, 42509, 32, 42512, -42513, 42560, 2097153, 42606, 42606, 42607, 0, + 42611, 32, 42612, 0, 42622, 32, 42623, 2097153, 42652, -42653, 42654, 0, + 42656, -42657, 42736, 0, 42738, 32, 42744, -42745, 42752, 32, 42775, -42776, + 42784, 32, 42786, 2097153, 42800, -42801, 42802, 2097153, 42864, -42865, 42873, 42874, + 42875, 42876, 42877, 7545, 42878, 2097153, 42888, 42888, 42889, 32, 42891, 42892, + 42893, 613, 42894, -42895, 42896, 2097153, 42900, -42901, 42902, 2097153, 42922, 614, + 42923, 604, 42924, 609, 42925, 620, 42926, 618, 42927, 42927, 42928, 670, + 42929, 647, 42930, 669, 42931, 43859, 42932, 2097153, 42936, -42937, 43000, 295, + 43001, -43002, 43010, 0, 43011, -43012, 43014, 0, 43015, -43016, 43019, 0, + 43020, -43021, 43043, 0, 43048, 32, 43052, -43053, 43062, 32, 43066, -43067, + 43124, 32, 43128, -43129, 43136, 0, 43138, -43139, 43188, 0, 43206, -43207, + 43214, 32, 43216, -43217, 43232, 0, 43250, -43251, 43256, 32, 43259, 43259, + 43260, 32, 43261, -43262, 43302, 0, 43310, 32, 43312, -43313, 43335, 0, + 43348, -43349, 43359, 32, 43360, -43361, 43392, 0, 43396, -43397, 43443, 0, + 43457, 32, 43470, -43471, 43486, 32, 43488, -43489, 43493, 0, 43494, -43495, + 43561, 0, 43575, -43576, 43587, 0, 43588, -43589, 43596, 0, 43598, -43599, + 43612, 32, 43616, -43617, 43639, 32, 43642, 43642, 43643, 0, 43646, -43647, + 43696, 0, 43697, 43697, 43698, 0, 43701, -43702, 43703, 0, 43705, -43706, + 43710, 0, 43712, 43712, 43713, 0, 43714, -43715, 43742, 32, 43744, -43745, + 43755, 0, 43760, 32, 43762, -43763, 43765, 0, 43767, -43768, 43867, 32, + 43868, -43869, 43888, -5025, 43968, -43969, 44003, 0, 44011, 32, 44012, 0, + 44014, -44015, 55296, 0, 57344, -57345, 64286, 0, 64287, -64288, 64297, 32, + 64298, -64299, 64434, 32, 64450, -64451, 64830, 32, 64832, -64833, 64976, 32, + 65008, -65009, 65020, 32, 65022, -65023, 65024, 0, 65040, 32, 65050, -65051, + 65056, 0, 65072, 32, 65107, 65107, 65108, 32, 65127, 65127, 65128, 32, + 65132, -65133, 65279, 0, 65280, 65280, 65281, 32, 65296, -65297, 65306, 32, + 65313, -65346, 65339, 32, 65345, -65346, 65371, 32, 65382, -65383, 65504, 32, + 65511, 65511, 65512, 32, 65519, -65520, 65529, 0, 65532, 32, 65536, -65537, + 65792, 32, 65795, -65796, 65847, 32, 65856, -65857, 65913, 32, 65930, -65931, + 65932, 32, 65935, 65935, 65936, 32, 65948, -65949, 65952, 32, 65953, -65954, + 66000, 32, 66045, 0, 66046, -66047, 66272, 0, 66273, -66274, 66422, 0, + 66427, -66428, 66463, 32, 66464, -66465, 66512, 32, 66513, -66514, 66560, -66601, + 66600, -66601, 66736, -66777, 66772, -66773, 66927, 32, 66928, -66929, 67671, 32, + 67672, -67673, 67703, 32, 67705, -67706, 67871, 32, 67872, -67873, 67903, 32, + 67904, -67905, 68097, 0, 68100, 68100, 68101, 0, 68103, -68104, 68108, 0, + 68112, -68113, 68152, 0, 68155, -68156, 68159, 0, 68160, -68161, 68176, 32, + 68185, -68186, 68223, 32, 68224, -68225, 68296, 32, 68297, -68298, 68325, 0, + 68327, -68328, 68336, 32, 68343, -68344, 68409, 32, 68416, -68417, 68505, 32, + 68509, -68510, 68736, -68801, 68787, -68788, 69632, 0, 69635, -69636, 69688, 0, + 69703, 32, 69710, -69711, 69759, 0, 69763, -69764, 69808, 0, 69819, 32, + 69821, 0, 69822, 32, 69826, -69827, 69888, 0, 69891, -69892, 69927, 0, + 69941, -69942, 69952, 32, 69956, -69957, 70003, 0, 70004, 32, 70006, -70007, + 70016, 0, 70019, -70020, 70067, 0, 70081, -70082, 70085, 32, 70090, 0, + 70093, 32, 70094, -70095, 70107, 32, 70108, 70108, 70109, 32, 70112, -70113, + 70188, 0, 70200, 32, 70206, 0, 70207, -70208, 70313, 32, 70314, -70315, + 70367, 0, 70379, -70380, 70400, 0, 70404, -70405, 70460, 0, 70461, 70461, + 70462, 0, 70469, -70470, 70471, 0, 70473, -70474, 70475, 0, 70478, -70479, + 70487, 0, 70488, -70489, 70498, 0, 70500, -70501, 70502, 0, 70509, -70510, + 70512, 0, 70517, -70518, 70709, 0, 70727, -70728, 70731, 32, 70736, -70737, + 70747, 32, 70748, 70748, 70749, 32, 70750, -70751, 70832, 0, 70852, -70853, + 70854, 32, 70855, -70856, 71087, 0, 71094, -71095, 71096, 0, 71105, 32, + 71128, -71129, 71132, 0, 71134, -71135, 71216, 0, 71233, 32, 71236, -71237, + 71264, 32, 71277, -71278, 71339, 0, 71352, -71353, 71453, 0, 71468, -71469, + 71484, 32, 71488, -71489, 71840, -71873, 71872, -71873, 72751, 0, 72759, 72759, + 72760, 0, 72768, 72768, 72769, 32, 72774, -72775, 72816, 32, 72818, -72819, + 72850, 0, 72872, 72872, 72873, 0, 72887, -72888, 74864, 32, 74869, -74870, + 92782, 32, 92784, -92785, 92912, 0, 92917, 32, 92918, -92919, 92976, 0, + 92983, 32, 92992, -92993, 92996, 32, 92998, -92999, 94033, 0, 94079, -94080, + 94095, 0, 94099, -94100, 113820, 32, 113821, 0, 113823, 32, 113824, 0, + 113828, -113829, 118784, 32, 119030, -119031, 119040, 32, 119079, -119080, 119081, 32, + 119141, 0, 119146, 32, 119149, 0, 119171, 32, 119173, 0, 119180, 32, + 119210, 0, 119214, 32, 119273, -119274, 119296, 32, 119362, 0, 119365, 32, + 119366, -119367, 119552, 32, 119639, -119640, 119808, -98, 119834, -119835, 119860, -98, + 119886, -119887, 119912, -98, 119938, -119939, 119964, 97, 119965, 119965, 119966, -100, + 119968, -119969, 119970, 103, 119971, -119972, 119973, -107, 119975, -119976, 119977, -111, + 119981, 119981, 119982, -116, 119990, -119991, 120016, -98, 120042, -120043, 120068, -98, + 120070, 120070, 120071, -101, 120075, -120076, 120077, -107, 120085, 120085, 120086, -116, + 120093, -120094, 120120, -98, 120122, 120122, 120123, -101, 120127, 120127, 120128, -106, + 120133, 120133, 120134, 111, 120135, -120136, 120138, -116, 120145, -120146, 120172, -98, + 120198, -120199, 120224, -98, 120250, -120251, 120276, -98, 120302, -120303, 120328, -98, + 120354, -120355, 120380, -98, 120406, -120407, 120432, -98, 120458, -120459, 120488, -946, + 120505, 952, 120506, -964, 120513, 32, 120514, -120515, 120531, 963, 120532, -120533, + 120539, 32, 120540, -120541, 120546, -946, 120563, 952, 120564, -964, 120571, 32, + 120572, -120573, 120589, 963, 120590, -120591, 120597, 32, 120598, -120599, 120604, -946, + 120621, 952, 120622, -964, 120629, 32, 120630, -120631, 120647, 963, 120648, -120649, + 120655, 32, 120656, -120657, 120662, -946, 120679, 952, 120680, -964, 120687, 32, + 120688, -120689, 120705, 963, 120706, -120707, 120713, 32, 120714, -120715, 120720, -946, + 120737, 952, 120738, -964, 120745, 32, 120746, -120747, 120763, 963, 120764, -120765, + 120771, 32, 120772, -120773, 120778, 989, 120779, -120780, 120832, 32, 121344, 0, + 121399, 32, 121403, 0, 121453, 32, 121461, 0, 121462, 32, 121476, 0, + 121477, 32, 121484, -121485, 121499, 0, 121504, 121504, 121505, 0, 121520, -121521, + 122880, 0, 122887, 122887, 122888, 0, 122905, -122906, 122907, 0, 122914, 122914, + 122915, 0, 122917, 122917, 122918, 0, 122923, -122924, 125136, 0, 125143, -125144, + 125184, -125219, 125218, -125219, 125252, 0, 125259, -125260, 125278, 32, 125280, -125281, + 126704, 32, 126706, -126707, 126976, 32, 127020, -127021, 127024, 32, 127124, -127125, + 127136, 32, 127151, -127152, 127153, 32, 127168, 127168, 127169, 32, 127184, 127184, + 127185, 32, 127222, -127223, 127248, 32, 127275, 99, 127276, 114, 127277, 32, + 127279, 127279, 127280, -98, 127306, 32, 127340, -127341, 127344, 32, 127405, -127406, + 127462, 32, 127490, -127491, 127552, 32, 127561, -127562, 127744, 32, 128723, -128724, + 128736, 32, 128749, -128750, 128752, 32, 128759, -128760, 128768, 32, 128884, -128885, + 128896, 32, 128981, -128982, 129024, 32, 129036, -129037, 129040, 32, 129096, -129097, + 129104, 32, 129114, -129115, 129120, 32, 129160, -129161, 129168, 32, 129198, -129199, + 129296, 32, 129311, 129311, 129312, 32, 129320, -129321, 129328, 32, 129329, -129330, + 129331, 32, 129343, 129343, 129344, 32, 129356, -129357, 129360, 32, 129375, -129376, + 129408, 32, 129426, -129427, 129472, 32, 129473, -129474, 131070, 32, 131072, -131073, + 196606, 32, 196608, -196609, 262142, 32, 262144, -262145, 327678, 32, 327680, -327681, + 393214, 32, 393216, -393217, 458750, 32, 458752, -458753, 524286, 32, 524288, -524289, + 589822, 32, 589824, -589825, 655358, 32, 655360, -655361, 720894, 32, 720896, -720897, + 786430, 32, 786432, -786433, 851966, 32, 851968, -851969, 917502, 32, 917504, 917504, + 917505, 0, 917506, -917507, 917536, 0, 917632, -917633, 917760, 0, 918000, -918001, + 983038, 32, 983040, -983041, 1048574, 32, 1048576, -1048577, 1114110, 32, 2147483647, 0}; + +static int16 to_lower_table[TABLE_SIZE] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 91, 92, 93, 94, + 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, + 241, 242, 243, 244, 245, 246, 215, 248, 249, 250, 251, 252, 253, 254, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 257, 257, 259, 259, 261, 261, 263, 263, 265, 265, + 267, 267, 269, 269, 271, 271, 273, 273, 275, 275, 277, 277, 279, 279, 281, 281, 283, 283, 285, + 285, 287, 287, 289, 289, 291, 291, 293, 293, 295, 295, 297, 297, 299, 299, 301, 301, 303, 303, + 105, 305, 307, 307, 309, 309, 311, 311, 312, 314, 314, 316, 316, 318, 318, 320, 320, 322, 322, + 324, 324, 326, 326, 328, 328, 329, 331, 331, 333, 333, 335, 335, 337, 337, 339, 339, 341, 341, + 343, 343, 345, 345, 347, 347, 349, 349, 351, 351, 353, 353, 355, 355, 357, 357, 359, 359, 361, + 361, 363, 363, 365, 365, 367, 367, 369, 369, 371, 371, 373, 373, 375, 375, 255, 378, 378, 380, + 380, 382, 382, 383, 384, 595, 387, 387, 389, 389, 596, 392, 392, 598, 599, 396, 396, 397, 477, + 601, 603, 402, 402, 608, 611, 405, 617, 616, 409, 409, 410, 411, 623, 626, 414, 629, 417, 417, + 419, 419, 421, 421, 640, 424, 424, 643, 426, 427, 429, 429, 648, 432, 432, 650, 651, 436, 436, + 438, 438, 658, 441, 441, 442, 443, 445, 445, 446, 447, 448, 449, 450, 451, 454, 454, 454, 457, + 457, 457, 460, 460, 460, 462, 462, 464, 464, 466, 466, 468, 468, 470, 470, 472, 472, 474, 474, + 476, 476, 477, 479, 479, 481, 481, 483, 483, 485, 485, 487, 487, 489, 489, 491, 491, 493, 493, + 495, 495, 496, 499, 499, 499, 501, 501, 405, 447, 505, 505, 507, 507, 509, 509, 511, 511, 513, + 513, 515, 515, 517, 517, 519, 519, 521, 521, 523, 523, 525, 525, 527, 527, 529, 529, 531, 531, + 533, 533, 535, 535, 537, 537, 539, 539, 541, 541, 543, 543, 414, 545, 547, 547, 549, 549, 551, + 551, 553, 553, 555, 555, 557, 557, 559, 559, 561, 561, 563, 563, 564, 565, 566, 567, 568, 569, + 11365, 572, 572, 410, 11366, 575, 576, 578, 578, 384, 649, 652, 583, 583, 585, 585, 587, 587, 589, + 589, 591, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, + 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, + 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, + 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, + 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, + 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, + 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, + 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, + 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, + 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, + 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, + 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, + 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, + 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, + 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, + 874, 875, 876, 877, 878, 879, 881, 881, 883, 883, 884, 885, 887, 887, 888, 889, 890, 891, 892, + 893, 894, 1011, 896, 897, 898, 899, 900, 901, 940, 903, 941, 942, 943, 907, 972, 909, 973, 974, + 912, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 930, + 963, 964, 965, 966, 967, 968, 969, 970, 971, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, + 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, + 969, 970, 971, 972, 973, 974, 983, 976, 977, 978, 979, 980, 981, 982, 983, 985, 985, 987, 987, + 989, 989, 991, 991, 993, 993, 995, 995, 997, 997, 999, 999, 1001, 1001, 1003, 1003, 1005, 1005, 1007, + 1007, 1008, 1009, 1010, 1011, 952, 1013, 1014, 1016, 1016, 1010, 1019, 1019, 1020, 891, 892, 893, 1104, 1105, + 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1072, 1073, 1074, 1075, 1076, + 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, + 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, + 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, + 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, + 1121, 1123, 1123, 1125, 1125, 1127, 1127, 1129, 1129, 1131, 1131, 1133, 1133, 1135, 1135, 1137, 1137, 1139, 1139, + 1141, 1141, 1143, 1143, 1145, 1145, 1147, 1147, 1149, 1149, 1151, 1151, 1153, 1153, 1154, 1155, 1156, 1157, 1158, + 1159, 1160, 1161, 1163, 1163, 1165, 1165, 1167, 1167, 1169, 1169, 1171, 1171, 1173, 1173, 1175, 1175, 1177, 1177, + 1179, 1179, 1181, 1181, 1183, 1183, 1185, 1185, 1187, 1187, 1189, 1189, 1191, 1191, 1193, 1193, 1195, 1195, 1197, + 1197, 1199, 1199, 1201, 1201, 1203, 1203, 1205, 1205, 1207, 1207, 1209, 1209, 1211, 1211, 1213, 1213, 1215, 1215, + 1231, 1218, 1218, 1220, 1220, 1222, 1222, 1224, 1224, 1226, 1226, 1228, 1228, 1230, 1230, 1231, 1233, 1233, 1235, + 1235, 1237, 1237, 1239, 1239, 1241, 1241, 1243, 1243, 1245, 1245, 1247, 1247, 1249, 1249, 1251, 1251, 1253, 1253, + 1255, 1255, 1257, 1257, 1259, 1259, 1261, 1261, 1263, 1263, 1265, 1265, 1267, 1267, 1269, 1269, 1271, 1271, 1273, + 1273, 1275, 1275, 1277, 1277, 1279, 1279}; + +static const int32 to_lower_ranges[] = { + 1280, 2097153, 1328, 1328, 1329, -1378, 1367, -1368, 4256, -11521, 4294, 4294, 4295, + 11559, 4296, -4297, 4301, 11565, 4302, -4303, 5024, -43889, 5104, -5113, 5110, -5111, + 7680, 2097153, 7830, -7831, 7838, 223, 7839, 2097153, 7936, -7937, 7944, -7937, 7952, + -7953, 7960, -7953, 7966, -7967, 7976, -7969, 7984, -7985, 7992, -7985, 8000, -8001, + 8008, -8001, 8014, -8015, 8025, 8017, 8026, 8026, 8027, 8019, 8028, 8028, 8029, + 8021, 8030, 8030, 8031, 8023, 8032, -8033, 8040, -8033, 8048, -8049, 8072, -8065, + 8080, -8081, 8088, -8081, 8096, -8097, 8104, -8097, 8112, -8113, 8120, -8113, 8122, + -8049, 8124, 8115, 8125, -8126, 8136, -8051, 8140, 8131, 8141, -8142, 8152, -8145, + 8154, -8055, 8156, -8157, 8168, -8161, 8170, -8059, 8172, 8165, 8173, -8174, 8184, + -8057, 8186, -8061, 8188, 8179, 8189, -8190, 8486, 969, 8487, -8488, 8490, 107, + 8491, 229, 8492, -8493, 8498, 8526, 8499, -8500, 8544, -8561, 8560, -8561, 8579, + 8580, 8581, -8582, 9398, -9425, 9424, -9425, 11264, -11313, 11311, -11312, 11360, 11361, + 11362, 619, 11363, 7549, 11364, 637, 11365, -11366, 11367, 11368, 11369, 11370, 11371, + 11372, 11373, 593, 11374, 625, 11375, 592, 11376, 594, 11377, 2097153, 11380, 11380, + 11381, 11382, 11383, -11384, 11390, -576, 11392, 2097153, 11492, -11493, 11499, 11500, 11501, + 11502, 11503, -11504, 11506, 11507, 11508, -11509, 42560, 2097153, 42606, -42607, 42624, 2097153, + 42652, -42653, 42786, 2097153, 42800, -42801, 42802, 2097153, 42864, -42865, 42873, 42874, 42875, + 42876, 42877, 7545, 42878, 2097153, 42888, -42889, 42891, 42892, 42893, 613, 42894, -42895, + 42896, 2097153, 42900, -42901, 42902, 2097153, 42922, 614, 42923, 604, 42924, 609, 42925, + 620, 42926, 618, 42927, 42927, 42928, 670, 42929, 647, 42930, 669, 42931, 43859, + 42932, 2097153, 42936, -42937, 65313, -65346, 65339, -65340, 66560, -66601, 66600, -66601, 66736, + -66777, 66772, -66773, 68736, -68801, 68787, -68788, 71840, -71873, 71872, -71873, 125184, -125219, + 125218, -125219, 2147483647, 0}; + +UnicodeSimpleCategory get_unicode_simple_category(uint32 code) { + auto it = std::upper_bound(std::begin(unicode_simple_category_ranges), std::end(unicode_simple_category_ranges), + (code << 5) + 30); + return static_cast<UnicodeSimpleCategory>(*(it - 1) & 31); +} + +/** + * Search pregenerated ranges of pairs for the replacement of specified character + */ +template <size_t N> +static uint32 binary_search_ranges(const int32 (&ranges)[N], uint32 code) { + if (code > 0x10ffff) { + return 0; + } + + int32 code_int = static_cast<int32>(code); + size_t l = 0, r = N; + while (l < r) { + size_t m = ((l + r + 2) >> 2) << 1; + if (ranges[m] <= code_int) { + l = m; + } else { + r = m - 2; + } + } + + int32 t = ranges[l + 1]; + if (t < 0) { + return code - ranges[l] + (~t); + } + if (t <= 0x10ffff) { + return t; + } + switch (t - 0x200000) { + case 0: + return (code & -2); + case 1: + return (code | 1); + case 2: + return ((code - 1) | 1); + default: + UNREACHABLE(); + return 0; + } +} + +uint32 prepare_search_character(uint32 code) { + if (code < TABLE_SIZE) { + return prepare_search_character_table[code]; + } else { + return binary_search_ranges(prepare_search_character_ranges, code); + } +} + +uint32 unicode_to_lower(uint32 code) { + if (code < TABLE_SIZE) { + return to_lower_table[code]; + } else { + return binary_search_ranges(to_lower_ranges, code); + } +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/unicode.h b/libs/tdlib/td/tdutils/td/utils/unicode.h new file mode 100644 index 0000000000..1c75397d6e --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/unicode.h @@ -0,0 +1,28 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" + +namespace td { + +enum class UnicodeSimpleCategory { Unknown, Letter, DecimalNumber, Number, Separator }; + +UnicodeSimpleCategory get_unicode_simple_category(uint32 code); + +/** + * Prepares unicode character for search, leaving only digits and lowercased letters. + * Return code of replacing character or 0 if the character should be skipped. + */ +uint32 prepare_search_character(uint32 code); + +/** + * Converts unicode character to lower case. + */ +uint32 unicode_to_lower(uint32 code); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/utf8.cpp b/libs/tdlib/td/tdutils/td/utils/utf8.cpp new file mode 100644 index 0000000000..50f82d6393 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/utf8.cpp @@ -0,0 +1,124 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/utf8.h" + +#include "td/utils/logging.h" // for UNREACHABLE +#include "td/utils/unicode.h" + +namespace td { + +bool check_utf8(CSlice str) { + const char *data = str.data(); + const char *data_end = data + str.size(); + do { + unsigned int a = static_cast<unsigned char>(*data++); + if ((a & 0x80) == 0) { + if (data == data_end + 1) { + return true; + } + continue; + } + +#define ENSURE(condition) \ + if (!(condition)) { \ + return false; \ + } + + ENSURE((a & 0x40) != 0); + + unsigned int b = static_cast<unsigned char>(*data++); + ENSURE((b & 0xc0) == 0x80); + if ((a & 0x20) == 0) { + ENSURE((a & 0x1e) > 0); + continue; + } + + unsigned int c = static_cast<unsigned char>(*data++); + ENSURE((c & 0xc0) == 0x80); + if ((a & 0x10) == 0) { + int x = (((a & 0x0f) << 6) | (b & 0x20)); + ENSURE(x != 0 && x != 0x360); // surrogates + continue; + } + + unsigned int d = static_cast<unsigned char>(*data++); + ENSURE((d & 0xc0) == 0x80); + if ((a & 0x08) == 0) { + int t = (((a & 0x07) << 6) | (b & 0x30)); + ENSURE(0 < t && t < 0x110); // end of unicode + continue; + } + + return false; +#undef ENSURE + } while (true); + + UNREACHABLE(); + return false; +} + +void append_utf8_character(string &str, uint32 ch) { + if (ch <= 0x7f) { + str.push_back(static_cast<char>(ch)); + } else if (ch <= 0x7ff) { + str.push_back(static_cast<char>(0xc0 | (ch >> 6))); // implementation-defined + str.push_back(static_cast<char>(0x80 | (ch & 0x3f))); + } else if (ch <= 0xffff) { + str.push_back(static_cast<char>(0xe0 | (ch >> 12))); // implementation-defined + str.push_back(static_cast<char>(0x80 | ((ch >> 6) & 0x3f))); + str.push_back(static_cast<char>(0x80 | (ch & 0x3f))); + } else { + str.push_back(static_cast<char>(0xf0 | (ch >> 18))); // implementation-defined + str.push_back(static_cast<char>(0x80 | ((ch >> 12) & 0x3f))); + str.push_back(static_cast<char>(0x80 | ((ch >> 6) & 0x3f))); + str.push_back(static_cast<char>(0x80 | (ch & 0x3f))); + } +} + +const unsigned char *next_utf8_unsafe(const unsigned char *ptr, uint32 *code) { + uint32 a = ptr[0]; + if ((a & 0x80) == 0) { + if (code) { + *code = a; + } + return ptr + 1; + } else if ((a & 0x20) == 0) { + if (code) { + *code = ((a & 0x1f) << 6) | (ptr[1] & 0x3f); + } + return ptr + 2; + } else if ((a & 0x10) == 0) { + if (code) { + *code = ((a & 0x0f) << 12) | ((ptr[1] & 0x3f) << 6) | (ptr[2] & 0x3f); + } + return ptr + 3; + } else if ((a & 0x08) == 0) { + if (code) { + *code = ((a & 0x07) << 18) | ((ptr[1] & 0x3f) << 12) | ((ptr[2] & 0x3f) << 6) | (ptr[3] & 0x3f); + } + return ptr + 4; + } + UNREACHABLE(); + if (code) { + *code = 0; + } + return ptr; +} + +string utf8_to_lower(Slice str) { + string result; + auto pos = str.ubegin(); + auto end = str.uend(); + while (pos != end) { + uint32 code; + pos = next_utf8_unsafe(pos, &code); + append_utf8_character(result, unicode_to_lower(code)); + } + return result; +} + +} // namespace td diff --git a/libs/tdlib/td/tdutils/td/utils/utf8.h b/libs/tdlib/td/tdutils/td/utils/utf8.h new file mode 100644 index 0000000000..6be1952c19 --- /dev/null +++ b/libs/tdlib/td/tdutils/td/utils/utf8.h @@ -0,0 +1,106 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/int_types.h" +#include "td/utils/Slice.h" + +namespace td { + +/// checks UTF-8 string for correctness +bool check_utf8(CSlice str); + +/// checks if a code unit is a first code unit of a UTF-8 character +inline bool is_utf8_character_first_code_unit(unsigned char c) { + return (c & 0xC0) != 0x80; +} + +/// returns length of UTF-8 string in characters +inline size_t utf8_length(Slice str) { + size_t result = 0; + for (auto c : str) { + result += is_utf8_character_first_code_unit(c); + } + return result; +} + +/// appends a Unicode character using UTF-8 encoding +void append_utf8_character(string &str, uint32 ch); + +/// moves pointer one UTF-8 character back +inline const unsigned char *prev_utf8_unsafe(const unsigned char *ptr) { + while (!is_utf8_character_first_code_unit(*--ptr)) { + // pass + } + return ptr; +} + +/// moves pointer one UTF-8 character forward and saves code of the skipped character in *code +const unsigned char *next_utf8_unsafe(const unsigned char *ptr, uint32 *code); + +/// truncates UTF-8 string to the given length in Unicode characters +template <class T> +T utf8_truncate(T str, size_t length) { + if (str.size() > length) { + for (size_t i = 0; i < str.size(); i++) { + if (is_utf8_character_first_code_unit(static_cast<unsigned char>(str[i]))) { + if (length == 0) { + return str.substr(0, i); + } else { + length--; + } + } + } + } + return str; +} + +/// truncates UTF-8 string to the given length given in UTF-16 code units +template <class T> +T utf8_utf16_truncate(T str, size_t length) { + for (size_t i = 0; i < str.size(); i++) { + auto c = static_cast<unsigned char>(str[i]); + if (is_utf8_character_first_code_unit(c)) { + if (length <= 0) { + return str.substr(0, i); + } else { + length--; + if (c >= 0xf0) { // >= 4 bytes in symbol => surrogaite pair + length--; + } + } + } + } + return str; +} + +template <class T> +T utf8_substr(T str, size_t offset) { + auto offset_pos = utf8_truncate(str, offset).size(); + return str.substr(offset_pos); +} + +template <class T> +T utf8_substr(T str, size_t offset, size_t length) { + return utf8_truncate(utf8_substr(str, offset), length); +} + +template <class T> +T utf8_utf16_substr(T str, size_t offset) { + auto offset_pos = utf8_utf16_truncate(str, offset).size(); + return str.substr(offset_pos); +} + +template <class T> +T utf8_utf16_substr(T str, size_t offset, size_t length) { + return utf8_utf16_truncate(utf8_utf16_substr(str, offset), length); +} + +/// Returns UTF-8 string converted to lower case. +string utf8_to_lower(Slice str); + +} // namespace td diff --git a/libs/tdlib/td/tdutils/test/Enumerator.cpp b/libs/tdlib/td/tdutils/test/Enumerator.cpp new file mode 100644 index 0000000000..b617485462 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/Enumerator.cpp @@ -0,0 +1,24 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Enumerator.h" +#include "td/utils/tests.h" + +TEST(Enumerator, simple) { + td::Enumerator<std::string> e; + auto b = e.add("b"); + auto a = e.add("a"); + auto d = e.add("d"); + auto c = e.add("c"); + ASSERT_STREQ(e.get(a), "a"); + ASSERT_STREQ(e.get(b), "b"); + ASSERT_STREQ(e.get(c), "c"); + ASSERT_STREQ(e.get(d), "d"); + ASSERT_EQ(a, e.add("a")); + ASSERT_EQ(b, e.add("b")); + ASSERT_EQ(c, e.add("c")); + ASSERT_EQ(d, e.add("d")); +} diff --git a/libs/tdlib/td/tdutils/test/HazardPointers.cpp b/libs/tdlib/td/tdutils/test/HazardPointers.cpp new file mode 100644 index 0000000000..36b0570530 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/HazardPointers.cpp @@ -0,0 +1,58 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/HazardPointers.h" +#include "td/utils/logging.h" +#include "td/utils/port/thread.h" +#include "td/utils/Random.h" +#include "td/utils/tests.h" + +#include <atomic> + +#if !TD_THREAD_UNSUPPORTED +TEST(HazardPointers, stress) { + struct Node { + std::atomic<std::string *> name_; + char pad[64]; + }; + int threads_n = 10; + std::vector<Node> nodes(threads_n); + td::HazardPointers<std::string> hazard_pointers(threads_n); + std::vector<td::thread> threads(threads_n); + int thread_id = 0; + for (auto &thread : threads) { + thread = td::thread([&, thread_id] { + auto holder = hazard_pointers.get_holder(thread_id, 0); + for (int i = 0; i < 1000000; i++) { + auto &node = nodes[td::Random::fast(0, threads_n - 1)]; + auto *str = holder.protect(node.name_); + if (str) { + CHECK(*str == "one" || *str == "twotwo"); + } + holder.clear(); + if (td::Random::fast(0, 5) == 0) { + std::string *new_str = new std::string(td::Random::fast(0, 1) == 0 ? "one" : "twotwo"); + if (node.name_.compare_exchange_strong(str, new_str, std::memory_order_acq_rel)) { + hazard_pointers.retire(thread_id, str); + } else { + delete new_str; + } + } + } + }); + thread_id++; + } + for (auto &thread : threads) { + thread.join(); + } + LOG(ERROR) << "Undeleted pointers: " << hazard_pointers.to_delete_size_unsafe(); + CHECK(static_cast<int>(hazard_pointers.to_delete_size_unsafe()) <= threads_n * threads_n); + for (int i = 0; i < threads_n; i++) { + hazard_pointers.retire(i); + } + CHECK(hazard_pointers.to_delete_size_unsafe() == 0); +} +#endif //!TD_THREAD_UNSUPPORTED diff --git a/libs/tdlib/td/tdutils/test/MpmcQueue.cpp b/libs/tdlib/td/tdutils/test/MpmcQueue.cpp new file mode 100644 index 0000000000..2da3f0cd3f --- /dev/null +++ b/libs/tdlib/td/tdutils/test/MpmcQueue.cpp @@ -0,0 +1,205 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/logging.h" +#include "td/utils/MpmcQueue.h" +#include "td/utils/port/thread.h" +#include "td/utils/tests.h" + +#include <algorithm> +#include <tuple> + +TEST(OneValue, simple) { + { + std::string x{"hello"}; + td::OneValue<std::string> value; + auto status = value.set_value(x); + CHECK(status); + CHECK(x.empty()); + status = value.get_value(x); + CHECK(status); + CHECK(x == "hello"); + } + { + td::OneValue<std::string> value; + std::string x; + auto status = value.get_value(x); + CHECK(!status); + CHECK(x.empty()); + std::string y{"hello"}; + status = value.set_value(y); + CHECK(!status); + CHECK(y == "hello"); + } +} + +#if !TD_THREAD_UNSUPPORTED +TEST(OneValue, stress) { + td::Stage run; + td::Stage check; + + std::string from; + bool set_status; + + std::string to; + bool get_status; + std::vector<td::thread> threads; + td::OneValue<std::string> value; + for (size_t i = 0; i < 2; i++) { + threads.push_back(td::thread([&, id = i] { + for (td::uint64 round = 1; round < 100000; round++) { + if (id == 0) { + value.reset(); + to = ""; + from = ""; + } + run.wait(round * 2); + if (id == 0) { + from = "hello"; + set_status = value.set_value(from); + } else { + get_status = value.get_value(to); + } + check.wait(round * 2); + if (id == 0) { + if (set_status) { + CHECK(get_status); + CHECK(from.empty()); + CHECK(to == "hello") << to; + } else { + CHECK(!get_status); + CHECK(from == "hello"); + CHECK(to.empty()); + } + } + } + })); + } + for (auto &thread : threads) { + thread.join(); + } +} +#endif //!TD_THREAD_UNSUPPORTED + +TEST(MpmcQueueBlock, simple) { + // Test doesn't work now and it is ok, try_pop, logic changed + return; + td::MpmcQueueBlock<std::string> block(2); + std::string x = "hello"; + using PushStatus = td::MpmcQueueBlock<std::string>::PushStatus; + using PopStatus = td::MpmcQueueBlock<std::string>::PopStatus; + auto push_status = block.push(x); + CHECK(push_status == PushStatus::Ok); + CHECK(x.empty()); + auto pop_status = block.pop(x); + CHECK(pop_status == PopStatus::Ok); + CHECK(x == "hello"); + pop_status = block.try_pop(x); + CHECK(pop_status == PopStatus::Empty); + x = "hello"; + push_status = block.push(x); + CHECK(push_status == PushStatus::Ok); + x = "hello"; + push_status = block.push(x); + CHECK(push_status == PushStatus::Closed); + CHECK(x == "hello"); + x = ""; + pop_status = block.try_pop(x); + CHECK(pop_status == PopStatus::Ok); + pop_status = block.try_pop(x); + CHECK(pop_status == PopStatus::Closed); +} + +TEST(MpmcQueue, simple) { + td::MpmcQueue<int> q(2, 1); + for (int t = 0; t < 2; t++) { + for (int i = 0; i < 100; i++) { + q.push(i, 0); + } + for (int i = 0; i < 100; i++) { + int x = q.pop(0); + CHECK(x == i) << x << " expected " << i; + } + } +} + +#if !TD_THREAD_UNSUPPORTED +TEST(MpmcQueue, multi_thread) { + size_t n = 10; + size_t m = 10; + struct Data { + size_t from{0}; + size_t value{0}; + }; + struct ThreadData { + std::vector<Data> v; + char pad[64]; + }; + td::MpmcQueue<Data> q(1024, n + m + 1); + std::vector<td::thread> n_threads(n); + std::vector<td::thread> m_threads(m); + std::vector<ThreadData> thread_data(m); + size_t thread_id = 0; + for (auto &thread : m_threads) { + thread = td::thread([&, thread_id] { + while (true) { + auto data = q.pop(thread_id); + if (data.value == 0) { + return; + } + thread_data[thread_id].v.push_back(data); + } + }); + thread_id++; + } + size_t qn = 100000; + for (auto &thread : n_threads) { + thread = td::thread([&, thread_id] { + for (size_t i = 0; i < qn; i++) { + Data data; + data.from = thread_id - m; + data.value = i + 1; + q.push(data, thread_id); + } + }); + thread_id++; + } + for (auto &thread : n_threads) { + thread.join(); + } + for (size_t i = 0; i < m; i++) { + Data data; + data.from = 0; + data.value = 0; + q.push(data, thread_id); + } + for (auto &thread : m_threads) { + thread.join(); + } + std::vector<Data> all; + for (size_t i = 0; i < m; i++) { + std::vector<size_t> from(n, 0); + for (auto &data : thread_data[i].v) { + all.push_back(data); + CHECK(data.value > from[data.from]); + from[data.from] = data.value; + } + } + CHECK(all.size() == n * qn) << all.size(); + std::sort(all.begin(), all.end(), + [](const auto &a, const auto &b) { return std::tie(a.from, a.value) < std::tie(b.from, b.value); }); + for (size_t i = 0; i < n * qn; i++) { + CHECK(all[i].from == i / qn); + CHECK(all[i].value == i % qn + 1); + } + LOG(ERROR) << "Undeleted pointers: " << q.hazard_pointers_to_delele_size_unsafe(); + CHECK(q.hazard_pointers_to_delele_size_unsafe() <= (n + m + 1) * (n + m + 1)); + for (size_t id = 0; id < n + m + 1; id++) { + q.gc(id); + } + CHECK(q.hazard_pointers_to_delele_size_unsafe() == 0) << q.hazard_pointers_to_delele_size_unsafe(); +} +#endif //!TD_THREAD_UNSUPPORTED diff --git a/libs/tdlib/td/tdutils/test/MpmcWaiter.cpp b/libs/tdlib/td/tdutils/test/MpmcWaiter.cpp new file mode 100644 index 0000000000..e27e217713 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/MpmcWaiter.cpp @@ -0,0 +1,117 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/MpmcWaiter.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/thread.h" +#include "td/utils/Random.h" +#include "td/utils/tests.h" + +#include <atomic> + +#if !TD_THREAD_UNSUPPORTED +TEST(MpmcWaiter, stress_one_one) { + td::Stage run; + td::Stage check; + + std::vector<td::thread> threads; + std::atomic<size_t> value; + size_t write_cnt = 10; + std::unique_ptr<td::MpmcWaiter> waiter; + size_t threads_n = 2; + for (size_t i = 0; i < threads_n; i++) { + threads.push_back(td::thread([&, id = static_cast<td::uint32>(i)] { + for (td::uint64 round = 1; round < 100000; round++) { + if (id == 0) { + value = 0; + waiter = std::make_unique<td::MpmcWaiter>(); + write_cnt = td::Random::fast(1, 10); + } + run.wait(round * threads_n); + if (id == 1) { + for (size_t i = 0; i < write_cnt; i++) { + value.store(i + 1, std::memory_order_relaxed); + waiter->notify(); + } + } else { + int yields = 0; + for (size_t i = 1; i <= write_cnt; i++) { + while (true) { + auto x = value.load(std::memory_order_relaxed); + if (x >= i) { + break; + } + yields = waiter->wait(yields, id); + } + yields = waiter->stop_wait(yields, id); + } + } + check.wait(round * threads_n); + } + })); + } + for (auto &thread : threads) { + thread.join(); + } +} +TEST(MpmcWaiter, stress) { + td::Stage run; + td::Stage check; + + std::vector<td::thread> threads; + size_t write_n; + size_t read_n; + std::atomic<size_t> write_pos; + std::atomic<size_t> read_pos; + size_t end_pos; + size_t write_cnt; + size_t threads_n = 20; + std::unique_ptr<td::MpmcWaiter> waiter; + for (size_t i = 0; i < threads_n; i++) { + threads.push_back(td::thread([&, id = static_cast<td::uint32>(i)] { + for (td::uint64 round = 1; round < 1000; round++) { + if (id == 0) { + write_n = td::Random::fast(1, 10); + read_n = td::Random::fast(1, 10); + write_cnt = td::Random::fast(1, 50); + end_pos = write_n * write_cnt; + write_pos = 0; + read_pos = 0; + waiter = std::make_unique<td::MpmcWaiter>(); + } + run.wait(round * threads_n); + if (id <= write_n) { + for (size_t i = 0; i < write_cnt; i++) { + if (td::Random::fast(0, 20) == 0) { + td::usleep_for(td::Random::fast(1, 300)); + } + write_pos.fetch_add(1, std::memory_order_relaxed); + waiter->notify(); + } + } else if (id > 10 && id - 10 <= read_n) { + int yields = 0; + while (true) { + auto x = read_pos.load(std::memory_order_relaxed); + if (x == end_pos) { + break; + } + if (x == write_pos.load(std::memory_order_relaxed)) { + yields = waiter->wait(yields, id); + continue; + } + yields = waiter->stop_wait(yields, id); + read_pos.compare_exchange_strong(x, x + 1, std::memory_order_relaxed); + } + } + check.wait(round * threads_n); + } + })); + } + for (auto &thread : threads) { + thread.join(); + } +} +#endif // !TD_THREAD_UNSUPPORTED diff --git a/libs/tdlib/td/tdutils/test/MpscLinkQueue.cpp b/libs/tdlib/td/tdutils/test/MpscLinkQueue.cpp new file mode 100644 index 0000000000..629e5b7223 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/MpscLinkQueue.cpp @@ -0,0 +1,115 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/MpscLinkQueue.h" +#include "td/utils/port/thread.h" +#include "td/utils/tests.h" + +class NodeX : public td::MpscLinkQueueImpl::Node { + public: + explicit NodeX(int value) : value_(value) { + } + td::MpscLinkQueueImpl::Node *to_mpsc_link_queue_node() { + return static_cast<td::MpscLinkQueueImpl::Node *>(this); + } + static NodeX *from_mpsc_link_queue_node(td::MpscLinkQueueImpl::Node *node) { + return static_cast<NodeX *>(node); + } + int value() { + return value_; + } + + private: + int value_; +}; +using QueueNode = td::MpscLinkQueueUniquePtrNode<NodeX>; + +QueueNode create_node(int value) { + return QueueNode(std::make_unique<NodeX>(value)); +} + +TEST(MpscLinkQueue, one_thread) { + td::MpscLinkQueue<QueueNode> queue; + + { + queue.push(create_node(1)); + queue.push(create_node(2)); + queue.push(create_node(3)); + td::MpscLinkQueue<QueueNode>::Reader reader; + queue.pop_all(reader); + queue.push(create_node(4)); + queue.pop_all(reader); + std::vector<int> v; + while (auto node = reader.read()) { + v.push_back(node.value().value()); + } + CHECK((v == std::vector<int>{1, 2, 3, 4})) << td::format::as_array(v); + + v.clear(); + queue.push(create_node(5)); + queue.pop_all(reader); + while (auto node = reader.read()) { + v.push_back(node.value().value()); + } + CHECK((v == std::vector<int>{5})) << td::format::as_array(v); + } + + { + queue.push_unsafe(create_node(3)); + queue.push_unsafe(create_node(2)); + queue.push_unsafe(create_node(1)); + queue.push_unsafe(create_node(0)); + td::MpscLinkQueue<QueueNode>::Reader reader; + queue.pop_all_unsafe(reader); + std::vector<int> v; + while (auto node = reader.read()) { + v.push_back(node.value().value()); + } + CHECK((v == std::vector<int>{3, 2, 1, 0})) << td::format::as_array(v); + } +} + +#if !TD_THREAD_UNSUPPORTED +TEST(MpscLinkQueue, multi_thread) { + td::MpscLinkQueue<QueueNode> queue; + int threads_n = 10; + int queries_n = 1000000; + std::vector<int> next_value(threads_n); + std::vector<td::thread> threads(threads_n); + int thread_i = 0; + for (auto &thread : threads) { + thread = td::thread([&, id = thread_i] { + for (int i = 0; i < queries_n; i++) { + queue.push(create_node(i * threads_n + id)); + } + }); + thread_i++; + } + + int active_threads = threads_n; + + td::MpscLinkQueue<QueueNode>::Reader reader; + while (active_threads) { + queue.pop_all(reader); + while (auto value = reader.read()) { + auto x = value.value().value(); + auto thread_id = x % threads_n; + x /= threads_n; + CHECK(next_value[thread_id] == x); + next_value[thread_id]++; + if (x + 1 == queries_n) { + active_threads--; + } + } + } + + for (auto &thread : threads) { + thread.join(); + } +} +#endif //!TD_THREAD_UNSUPPORTED diff --git a/libs/tdlib/td/tdutils/test/OrderedEventsProcessor.cpp b/libs/tdlib/td/tdutils/test/OrderedEventsProcessor.cpp new file mode 100644 index 0000000000..6a5a20015f --- /dev/null +++ b/libs/tdlib/td/tdutils/test/OrderedEventsProcessor.cpp @@ -0,0 +1,36 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/OrderedEventsProcessor.h" +#include "td/utils/Random.h" +#include "td/utils/tests.h" + +#include <algorithm> +#include <utility> +#include <vector> + +TEST(OrderedEventsProcessor, random) { + int d = 5001; + int n = 1000000; + int offset = 1000000; + std::vector<std::pair<int, int>> v; + for (int i = 0; i < n; i++) { + auto shift = td::Random::fast(0, 1) ? td::Random::fast(0, d) : td::Random::fast(0, 1) * d; + v.push_back({i + shift, i + offset}); + } + std::sort(v.begin(), v.end()); + + td::OrderedEventsProcessor<int> processor(offset); + int next_pos = offset; + for (auto p : v) { + int seq_no = p.second; + processor.add(seq_no, seq_no, [&](auto seq_no, int x) { + ASSERT_EQ(x, next_pos); + next_pos++; + }); + } + ASSERT_EQ(next_pos, n + offset); +} diff --git a/libs/tdlib/td/tdutils/test/SharedObjectPool.cpp b/libs/tdlib/td/tdutils/test/SharedObjectPool.cpp new file mode 100644 index 0000000000..61d956f4e6 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/SharedObjectPool.cpp @@ -0,0 +1,96 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/logging.h" +#include "td/utils/SharedObjectPool.h" +#include "td/utils/tests.h" + +#include <memory> + +TEST(AtomicRefCnt, simple) { + td::detail::AtomicRefCnt cnt{0}; + cnt.inc(); + cnt.inc(); + CHECK(!cnt.dec()); + cnt.inc(); + CHECK(!cnt.dec()); + CHECK(cnt.dec()); + cnt.inc(); + CHECK(cnt.dec()); +} + +template <class T, class D> +using Ptr = td::detail::SharedPtr<T, D>; +class Deleter { + public: + template <class T> + void operator()(T *t) { + std::default_delete<T>()(t); + was_delete() = true; + } + static bool &was_delete() { + static bool flag = false; + return flag; + } +}; + +TEST(SharedPtr, simple) { + CHECK(!Deleter::was_delete()); + Ptr<std::string, Deleter> ptr = Ptr<std::string, Deleter>::create("hello"); + auto ptr2 = ptr; + CHECK(*ptr == "hello"); + CHECK(*ptr2 == "hello"); + ptr.reset(); + CHECK(*ptr2 == "hello"); + CHECK(ptr.empty()); + Ptr<std::string, Deleter> ptr3 = std::move(ptr2); + CHECK(ptr2.empty()); + CHECK(*ptr3 == "hello"); + ptr = ptr3; + CHECK(*ptr3 == "hello"); + ptr3.reset(); + CHECK(*ptr == "hello"); + ptr2 = std::move(ptr); + CHECK(ptr.empty()); + CHECK(*ptr2 == "hello"); + ptr2 = ptr2; + CHECK(*ptr2 == "hello"); + CHECK(!Deleter::was_delete()); + ptr2.reset(); + CHECK(Deleter::was_delete()); + CHECK(ptr2.empty()); +} + +TEST(SharedObjectPool, simple) { + class Node { + public: + Node() { + cnt()++; + }; + ~Node() { + cnt()--; + } + static int &cnt() { + static int cnt_ = 0; + return cnt_; + } + }; + { + td::SharedObjectPool<Node> pool; + pool.alloc(); + pool.alloc(); + pool.alloc(); + pool.alloc(); + pool.alloc(); + CHECK(Node::cnt() == 0); + CHECK(pool.total_size() == 1); + CHECK(pool.calc_free_size() == 1); + pool.alloc(), pool.alloc(), pool.alloc(); + CHECK(pool.total_size() == 3); + CHECK(pool.calc_free_size() == 3); + } + CHECK(Node::cnt() == 0); +} diff --git a/libs/tdlib/td/tdutils/test/crypto.cpp b/libs/tdlib/td/tdutils/test/crypto.cpp new file mode 100644 index 0000000000..faf4ef61a4 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/crypto.cpp @@ -0,0 +1,166 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/base64.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/Slice.h" +#include "td/utils/tests.h" + +#include <limits> + +static td::vector<td::string> strings{"", "1", "short test string", td::string(1000000, 'a')}; + +#if TD_HAVE_OPENSSL +TEST(Crypto, AesCtrState) { + td::vector<td::uint32> answers1{0u, 1141589763u, 596296607u, 3673001485u, 2302125528u, + 330967191u, 2047392231u, 3537459563u, 307747798u, 2149598133u}; + td::vector<td::uint32> answers2{0u, 2053451992u, 1384063362u, 3266188502u, 2893295118u, + 780356167u, 1904947434u, 2043402406u, 472080809u, 1807109488u}; + + std::size_t i = 0; + for (auto length : {0, 1, 31, 32, 33, 9999, 10000, 10001, 999999, 1000001}) { + td::uint32 seed = length; + td::string s(length, '\0'); + for (auto &c : s) { + seed = seed * 123457567u + 987651241u; + c = static_cast<char>((seed >> 23) & 255); + } + + td::UInt256 key; + for (auto &c : key.raw) { + seed = seed * 123457567u + 987651241u; + c = (seed >> 23) & 255; + } + td::UInt128 iv; + for (auto &c : iv.raw) { + seed = seed * 123457567u + 987651241u; + c = (seed >> 23) & 255; + } + + td::AesCtrState state; + state.init(key, iv); + td::string t(length, '\0'); + state.encrypt(s, t); + ASSERT_EQ(answers1[i], td::crc32(t)); + state.init(key, iv); + state.decrypt(t, t); + ASSERT_STREQ(s, t); + + for (auto &c : iv.raw) { + c = 0xFF; + } + state.init(key, iv); + state.encrypt(s, t); + ASSERT_EQ(answers2[i], td::crc32(t)); + + i++; + } +} + +TEST(Crypto, Sha256State) { + for (auto length : {0, 1, 31, 32, 33, 9999, 10000, 10001, 999999, 1000001}) { + auto s = td::rand_string(std::numeric_limits<char>::min(), std::numeric_limits<char>::max(), length); + td::UInt256 baseline; + td::sha256(s, td::MutableSlice(baseline.raw, 32)); + + td::Sha256State state; + td::sha256_init(&state); + auto v = td::rand_split(s); + for (auto &x : v) { + td::sha256_update(x, &state); + } + td::UInt256 result; + td::sha256_final(&state, td::MutableSlice(result.raw, 32)); + ASSERT_TRUE(baseline == result); + } +} + +TEST(Crypto, PBKDF) { + td::vector<td::string> passwords{"", "qwerty", std::string(1000, 'a')}; + td::vector<td::string> salts{"", "qwerty", std::string(1000, 'a')}; + td::vector<int> iteration_counts{1, 2, 1000}; + td::vector<td::Slice> answers{ + "984LZT0tcqQQjPWr6RL/3Xd2Ftu7J6cOggTzri0Pb60=", "lzmEEdaupDp3rO+SImq4J41NsGaL0denanJfdoCsRcU=", + "T8WKIcEAzhg1uPmZHXOLVpZdFLJOF2H73/xprF4LZno=", "NHxAnMhPOATsb1wV0cGDlAIs+ofzI6I4I8eGJeWN9Qw=", + "fjYi7waEPjbVYEuZ61/Nm2hbk/vRdShoJoXg4Ygnqe4=", "GhW6e95hGJSf+ID5IrSbvzWyBZ1l35A+UoL55Uh/njk=", + "BueLDpqSCEc0GWk83WgMwz3UsWwfvVKcvllETSB/Yq8=", "hgHgJZNWRh78PyPdVJsK8whgHOHQbNQiyaTuGDX2IFo=", + "T2xdyNT1GlcA4+MVNzOe7NCgSAAzNkanNsmuoSr+4xQ=", "/f6t++GUPE+e63+0TrlInL+UsmzRSAAFopa8BBBmb2w=", + "8Zn98QEAKS9wPOUlN09+pfm0SWs1IGeQxQkNMT/1k48=", "sURLQ/6UX/KVYedyQB21oAtMJ+STZ4iwpxfQtqmWkLw=", + "T9t/EJXFpPs2Lhca7IVGphTC/OdEloPMHw1UhDnXcyQ=", "TIrtN05E9KQL6Lp/wjtbsFS+KkWZ8jlGK0ErtaoitOg=", + "+1KcMBjyUNz5VMaIfE5wkGwS6I+IQ5FhK+Ou2HgtVoQ=", "h36ci1T0vGllCl/xJxq6vI7n28Bg40dilzWOKg6Jt8k=", + "9uwsHJsotTiTqqCYftN729Dg7QI2BijIjV2MvSEUAeE=", "/l+vd/XYgbioh1SfLMaGRr13udmY6TLSlG4OYmytwGU=", + "7qfZZBbMRLtgjqq7GHgWa/UfXPajW8NXpJ6/T3P1rxI=", "ufwz94p28WnoOFdbrb1oyQEzm/v0CV2b0xBVxeEPJGA=", + "T/PUUBX2vGMUsI6httlhbMHlGPMvqFBNzayU5voVlaw=", "viMvsvTg9GfQymF3AXZ8uFYTDa3qLrqJJk9w/74iZfg=", + "HQF+rOZMW4DAdgZz8kAMe28eyIi0rs3a3u/mUeGPNfs=", "7lBVA+GnSxWF/eOo+tyyTB7niMDl1MqP8yzo+xnHTyw=", + "aTWb7HQAxaTKhSiRPY3GuM1GVmq/FPuwWBU/TUpdy70=", "fbg8M/+Ht/oU+UAZ4dQcGPo+wgCCHaA+GM4tm5jnWcY=", + "DJbCGFMIR/5neAlpda8Td5zftK4NGekVrg2xjrKW/4c="}; + + std::size_t pos = 0; + for (auto &password : passwords) { + for (auto &salt : salts) { + for (auto &iteration_count : iteration_counts) { + char result[32]; + td::pbkdf2_sha256(password, salt, iteration_count, {result, 32}); + ASSERT_STREQ(answers[pos], td::base64_encode({result, 32})); + pos++; + } + } + } +} + +TEST(Crypto, sha1) { + td::vector<td::Slice> answers{"2jmj7l5rSw0yVb/vlWAYkK/YBwk=", "NWoZK3kTsExUV00Ywo1G5jlUKKs=", + "uRysQwoax0pNJeBC3+zpQzJy1rA=", "NKqXPNTE2qT2Husr260nMWU0AW8="}; + + for (std::size_t i = 0; i < strings.size(); i++) { + unsigned char output[20]; + td::sha1(strings[i], output); + ASSERT_STREQ(answers[i], td::base64_encode(td::Slice(output, 20))); + } +} + +TEST(Crypto, sha256) { + td::vector<td::Slice> answers{ + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", "a4ayc/80/OGda4BO/1o/V0etpOqiLx1JwB5S3beHW0s=", + "yPMaY7Q8PKPwCsw64UnDD5mhRcituEJgzLZMvr0O8pY=", "zcduXJkU+5KBocfihNc+Z/GAmkiklyAOBG05zMcRLNA="}; + + for (std::size_t i = 0; i < strings.size(); i++) { + td::string output(32, '\0'); + td::sha256(strings[i], output); + ASSERT_STREQ(answers[i], td::base64_encode(output)); + } +} + +TEST(Crypto, md5) { + td::vector<td::Slice> answers{ + "1B2M2Y8AsgTpgAmY7PhCfg==", "xMpCOKC5I4INzFCab3WEmw==", "vwBninYbDRkgk+uA7GMiIQ==", "dwfWrk4CfHDuoqk1wilvIQ=="}; + + for (std::size_t i = 0; i < strings.size(); i++) { + td::string output(16, '\0'); + td::md5(strings[i], output); + ASSERT_STREQ(answers[i], td::base64_encode(output)); + } +} +#endif + +#if TD_HAVE_ZLIB +TEST(Crypto, crc32) { + td::vector<td::uint32> answers{0u, 2212294583u, 3013144151u, 3693461436u}; + + for (std::size_t i = 0; i < strings.size(); i++) { + ASSERT_EQ(answers[i], td::crc32(strings[i])); + } +} +#endif + +TEST(Crypto, crc64) { + td::vector<td::uint64> answers{0ull, 3039664240384658157ull, 17549519902062861804ull, 8794730974279819706ull}; + + for (std::size_t i = 0; i < strings.size(); i++) { + ASSERT_EQ(answers[i], td::crc64(strings[i])); + } +} diff --git a/libs/tdlib/td/tdutils/test/filesystem.cpp b/libs/tdlib/td/tdutils/test/filesystem.cpp new file mode 100644 index 0000000000..a0a92c14eb --- /dev/null +++ b/libs/tdlib/td/tdutils/test/filesystem.cpp @@ -0,0 +1,41 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/filesystem.h" +#include "td/utils/tests.h" + +TEST(Misc, clean_filename) { + using td::clean_filename; + ASSERT_STREQ(clean_filename("-1234567"), "-1234567"); + ASSERT_STREQ(clean_filename(".git"), "git"); + ASSERT_STREQ(clean_filename("../../.git"), "git"); + ASSERT_STREQ(clean_filename(".././.."), ""); + ASSERT_STREQ(clean_filename("../"), ""); + ASSERT_STREQ(clean_filename(".."), ""); + ASSERT_STREQ(clean_filename("test/git/ as dsa . a"), "as dsa.a"); + ASSERT_STREQ(clean_filename(" . "), ""); + ASSERT_STREQ(clean_filename("!@#$%^&*()_+-=[]{;|:\"}'<>?,.`~"), "!@#$%^ ()_+-=[]{; } ,.~"); + ASSERT_STREQ(clean_filename("!@#$%^&*()_+-=[]{}\\|:\";'<>?,.`~"), "; ,.~"); + ASSERT_STREQ(clean_filename("عرفها بعد قد. هذا مع تاريخ اليميني واندونيسيا،, لعدم تاريخ لهيمنة الى"), + "عرفها بعد قد.هذا مع تاريخ اليميني"); + ASSERT_STREQ( + clean_filename( + "012345678901234567890123456789012345678901234567890123456789adsasdasdsaa.01234567890123456789asdasdasdasd"), + "012345678901234567890123456789012345678901234567890123456789.01234567890123456789"); + ASSERT_STREQ(clean_filename("01234567890123456789012345678901234567890123456789<>*?: <>*?:0123456789adsasdasdsaa. " + "0123456789`<><<>><><>0123456789asdasdasdasd"), + "01234567890123456789012345678901234567890123456789.0123456789"); + ASSERT_STREQ(clean_filename("01234567890123456789012345678901234567890123456789<>*?: <>*?:0123456789adsasdasdsaa. " + "0123456789`<><><>0123456789asdasdasdasd"), + "01234567890123456789012345678901234567890123456789.0123456789 012"); + ASSERT_STREQ(clean_filename("C:/document.tar.gz"), "document.tar.gz"); + ASSERT_STREQ(clean_filename("test...."), "test"); + ASSERT_STREQ(clean_filename("....test"), "test"); + ASSERT_STREQ(clean_filename("test.exe...."), "test.exe"); // extension has changed + ASSERT_STREQ(clean_filename("test.exe01234567890123456789...."), + "test.exe01234567890123456789"); // extension may be more then 20 characters + ASSERT_STREQ(clean_filename("....test....asdf"), "test.asdf"); +} diff --git a/libs/tdlib/td/tdutils/test/gzip.cpp b/libs/tdlib/td/tdutils/test/gzip.cpp new file mode 100644 index 0000000000..e4bd81eb0d --- /dev/null +++ b/libs/tdlib/td/tdutils/test/gzip.cpp @@ -0,0 +1,113 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/buffer.h" +#include "td/utils/ByteFlow.h" +#include "td/utils/Gzip.h" +#include "td/utils/GzipByteFlow.h" +#include "td/utils/logging.h" +#include "td/utils/Status.h" +#include "td/utils/tests.h" + +static void encode_decode(td::string s) { + auto r = td::gzencode(s, 2); + ASSERT_TRUE(!r.empty()); + if (r.empty()) { + return; + } + auto new_s = td::gzdecode(r.as_slice()); + ASSERT_TRUE(!new_s.empty()); + if (new_s.empty()) { + return; + } + ASSERT_EQ(s, new_s.as_slice().str()); +} + +TEST(Gzip, gzencode_gzdecode) { + auto str = td::rand_string(0, 127, 1000); + encode_decode(str); + str = td::rand_string('a', 'z', 1000000); + encode_decode(str); + str = td::string(1000000, 'a'); + encode_decode(str); +} + +TEST(Gzip, flow) { + auto str = td::rand_string('a', 'z', 1000000); + auto parts = td::rand_split(str); + + auto input_writer = td::ChainBufferWriter::create_empty(); + auto input = input_writer.extract_reader(); + td::ByteFlowSource source(&input); + td::GzipByteFlow gzip_flow(td::Gzip::Encode); + gzip_flow = td::GzipByteFlow(td::Gzip::Encode); + td::ByteFlowSink sink; + + source >> gzip_flow >> sink; + + ASSERT_TRUE(!sink.is_ready()); + for (auto &part : parts) { + input_writer.append(part); + source.wakeup(); + } + ASSERT_TRUE(!sink.is_ready()); + source.close_input(td::Status::OK()); + ASSERT_TRUE(sink.is_ready()); + ASSERT_TRUE(sink.status().is_ok()); + auto res = sink.result()->move_as_buffer_slice().as_slice().str(); + ASSERT_TRUE(!res.empty()); + ASSERT_EQ(td::gzencode(str, 2).as_slice().str(), res); +} +TEST(Gzip, flow_error) { + auto str = td::rand_string('a', 'z', 1000000); + auto zip = td::gzencode(str).as_slice().str(); + zip.resize(zip.size() - 1); + auto parts = td::rand_split(zip); + + auto input_writer = td::ChainBufferWriter(); + auto input = input_writer.extract_reader(); + td::ByteFlowSource source(&input); + td::GzipByteFlow gzip_flow(td::Gzip::Decode); + td::ByteFlowSink sink; + + source >> gzip_flow >> sink; + + ASSERT_TRUE(!sink.is_ready()); + for (auto &part : parts) { + input_writer.append(part); + source.wakeup(); + } + ASSERT_TRUE(!sink.is_ready()); + source.close_input(td::Status::OK()); + ASSERT_TRUE(sink.is_ready()); + ASSERT_TRUE(!sink.status().is_ok()); +} + +TEST(Gzip, encode_decode_flow) { + auto str = td::rand_string('a', 'z', 1000000); + auto parts = td::rand_split(str); + auto input_writer = td::ChainBufferWriter::create_empty(); + auto input = input_writer.extract_reader(); + td::ByteFlowSource source(&input); + td::GzipByteFlow gzip_encode_flow(td::Gzip::Encode); + td::GzipByteFlow gzip_decode_flow(td::Gzip::Decode); + td::GzipByteFlow gzip_encode_flow2(td::Gzip::Encode); + td::GzipByteFlow gzip_decode_flow2(td::Gzip::Decode); + td::ByteFlowSink sink; + source >> gzip_encode_flow >> gzip_decode_flow >> gzip_encode_flow2 >> gzip_decode_flow2 >> sink; + + ASSERT_TRUE(!sink.is_ready()); + for (auto &part : parts) { + input_writer.append(part); + source.wakeup(); + } + ASSERT_TRUE(!sink.is_ready()); + source.close_input(td::Status::OK()); + ASSERT_TRUE(sink.is_ready()); + LOG_IF(ERROR, sink.status().is_error()) << sink.status(); + ASSERT_TRUE(sink.status().is_ok()); + ASSERT_EQ(str, sink.result()->move_as_buffer_slice().as_slice().str()); +} diff --git a/libs/tdlib/td/tdutils/test/heap.cpp b/libs/tdlib/td/tdutils/test/heap.cpp new file mode 100644 index 0000000000..0dcfcf98ff --- /dev/null +++ b/libs/tdlib/td/tdutils/test/heap.cpp @@ -0,0 +1,178 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/tests.h" + +#include "td/utils/common.h" +#include "td/utils/Heap.h" +#include "td/utils/logging.h" +#include "td/utils/Random.h" + +#include <algorithm> +#include <cstdio> +#include <cstdlib> +#include <set> +#include <utility> + +REGISTER_TESTS(heap) + +using namespace td; + +TEST(Heap, sort_random_perm) { + int n = 1000000; + std::vector<int> v(n); + for (int i = 0; i < n; i++) { + v[i] = i; + } + std::srand(123); + std::random_shuffle(v.begin(), v.end()); + std::vector<HeapNode> nodes(n); + KHeap<int> kheap; + for (int i = 0; i < n; i++) { + kheap.insert(v[i], &nodes[i]); + } + for (int i = 0; i < n; i++) { + ASSERT_EQ(i, kheap.top_key()); + kheap.pop(); + } +}; + +class CheckedHeap { + public: + void set_max_size(int max_size) { + nodes.resize(max_size); + free_ids.resize(max_size); + rev_ids.resize(max_size); + for (int i = 0; i < max_size; i++) { + free_ids[i] = max_size - i - 1; + nodes[i].value = i; + } + } + static void xx(int key, const HeapNode *heap_node) { + const Node *node = static_cast<const Node *>(heap_node); + std::fprintf(stderr, "(%d;%d)", node->key, node->value); + } + void check() const { + for (auto p : set_heap) { + std::fprintf(stderr, "(%d;%d)", p.first, p.second); + } + std::fprintf(stderr, "\n"); + kheap.for_each(xx); + std::fprintf(stderr, "\n"); + kheap.check(); + } + int random_id() const { + CHECK(!empty()); + return ids[Random::fast(0, static_cast<int>(ids.size() - 1))]; + } + size_t size() const { + return ids.size(); + } + bool empty() const { + return ids.empty(); + } + + int top_key() const { + CHECK(!empty()); + int res = set_heap.begin()->first; + ASSERT_EQ(set_heap.size(), kheap.size()); + ASSERT_EQ(res, kheap.top_key()); + return res; + } + int insert(int key) { + // std::fprintf(stderr, "insert %d\n", key); + int id; + if (free_ids.empty()) { + UNREACHABLE(); + id = static_cast<int>(nodes.size()); + nodes.emplace_back(key, id); + rev_ids.push_back(-1); + } else { + id = free_ids.back(); + free_ids.pop_back(); + nodes[id].key = key; + } + rev_ids[id] = static_cast<int>(ids.size()); + ids.push_back(id); + kheap.insert(key, &nodes[id]); + set_heap.emplace(key, id); + return id; + } + void fix_key(int new_key, int id) { + // std::fprintf(stderr, "fix key %d %d (old_key = %d)\n", new_key, id, nodes[id].key); + set_heap.erase(std::make_pair(nodes[id].key, id)); + nodes[id].key = new_key; + kheap.fix(new_key, &nodes[id]); + set_heap.emplace(new_key, id); + } + void erase(int id) { + // std::fprintf(stderr, "erase %d\n", id); + int pos = rev_ids[id]; + CHECK(pos != -1); + ids[pos] = ids.back(); + rev_ids[ids[pos]] = pos; + ids.pop_back(); + rev_ids[id] = -1; + free_ids.push_back(id); + + kheap.erase(&nodes[id]); + set_heap.erase(std::make_pair(nodes[id].key, id)); + } + void pop() { + // std::fprintf(stderr, "pop\n"); + CHECK(!empty()); + Node *node = static_cast<Node *>(kheap.pop()); + int id = node->value; + ASSERT_EQ(node->key, set_heap.begin()->first); + + int pos = rev_ids[id]; + CHECK(pos != -1); + ids[pos] = ids.back(); + rev_ids[ids[pos]] = pos; + ids.pop_back(); + rev_ids[id] = -1; + free_ids.push_back(id); + + set_heap.erase(std::make_pair(nodes[id].key, id)); + } + + private: + struct Node : public HeapNode { + Node() = default; + Node(int key, int value) : key(key), value(value) { + } + int key = 0; + int value = 0; + }; + vector<int> ids; + vector<int> rev_ids; + vector<int> free_ids; + vector<Node> nodes; + std::set<std::pair<int, int>> set_heap; + KHeap<int> kheap; +}; + +TEST(Heap, random_events) { + CheckedHeap heap; + heap.set_max_size(1000); + for (int i = 0; i < 300000; i++) { + if (!heap.empty()) { + heap.top_key(); + } + + int x = Random::fast(0, 4); + if (heap.empty() || (x < 2 && heap.size() < 1000)) { + heap.insert(Random::fast(0, 99)); + } else if (x < 3) { + heap.fix_key(Random::fast(0, 99), heap.random_id()); + } else if (x < 4) { + heap.erase(heap.random_id()); + } else if (x < 5) { + heap.pop(); + } + // heap.check(); + } +} diff --git a/libs/tdlib/td/tdutils/test/json.cpp b/libs/tdlib/td/tdutils/test/json.cpp new file mode 100644 index 0000000000..deca81a791 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/json.cpp @@ -0,0 +1,94 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/tests.h" + +#include "td/utils/JsonBuilder.h" +#include "td/utils/logging.h" +#include "td/utils/StringBuilder.h" + +#include <tuple> +#include <utility> + +REGISTER_TESTS(json) + +using namespace td; + +static void decode_encode(string str, string result = "") { + auto str_copy = str; + auto r_value = json_decode(str_copy); + ASSERT_TRUE(r_value.is_ok()); + if (r_value.is_error()) { + LOG(INFO) << r_value.error(); + return; + } + auto new_str = json_encode<string>(r_value.ok()); + if (result.empty()) { + result = str; + } + ASSERT_EQ(result, new_str); +} + +TEST(JSON, array) { + char tmp[1000]; + StringBuilder sb({tmp, sizeof(tmp)}); + JsonBuilder jb(std::move(sb)); + jb.enter_value().enter_array() << "Hello" << -123; + ASSERT_EQ(jb.string_builder().is_error(), false); + auto encoded = jb.string_builder().as_cslice().str(); + ASSERT_EQ("[\"Hello\",-123]", encoded); + decode_encode(encoded); +} +TEST(JSON, object) { + char tmp[1000]; + StringBuilder sb({tmp, sizeof(tmp)}); + JsonBuilder jb(std::move(sb)); + auto c = jb.enter_object(); + c << std::tie("key", "value"); + c << std::make_pair("1", 2); + c.leave(); + ASSERT_EQ(jb.string_builder().is_error(), false); + auto encoded = jb.string_builder().as_cslice().str(); + ASSERT_EQ("{\"key\":\"value\",\"1\":2}", encoded); + decode_encode(encoded); +} + +TEST(JSON, nested) { + char tmp[1000]; + StringBuilder sb({tmp, sizeof(tmp)}); + JsonBuilder jb(std::move(sb)); + { + auto a = jb.enter_array(); + a << 1; + { a.enter_value().enter_array() << 2; } + a << 3; + } + ASSERT_EQ(jb.string_builder().is_error(), false); + auto encoded = jb.string_builder().as_cslice().str(); + ASSERT_EQ("[1,[2],3]", encoded); + decode_encode(encoded); +} + +TEST(JSON, kphp) { + decode_encode("[]"); + decode_encode("[[]]"); + decode_encode("{}"); + decode_encode("{}"); + decode_encode("\"\\n\""); + decode_encode( + "\"" + "some long string \\t \\r \\\\ \\n \\f \\\" " + "\\u1234" + "\""); + decode_encode( + "{\"keyboard\":[[\"\\u2022 abcdefg\"],[\"\\u2022 hijklmnop\"],[\"\\u2022 " + "qrstuvwxyz\"]],\"one_time_keyboard\":true}"); + decode_encode( + " \n { \"keyboard\" : \n [[ \"\\u2022 abcdefg\" ] , \n [ \"\\u2022 hijklmnop\" \n ],[ \n \"\\u2022 " + "qrstuvwxyz\"]], \n \"one_time_keyboard\"\n:\ntrue\n}\n \n", + "{\"keyboard\":[[\"\\u2022 abcdefg\"],[\"\\u2022 hijklmnop\"],[\"\\u2022 " + "qrstuvwxyz\"]],\"one_time_keyboard\":true}"); +} diff --git a/libs/tdlib/td/tdutils/test/misc.cpp b/libs/tdlib/td/tdutils/test/misc.cpp new file mode 100644 index 0000000000..dd1f1ec457 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/misc.cpp @@ -0,0 +1,262 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/base64.h" +#include "td/utils/HttpUrl.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/EventFd.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/port/path.h" +#include "td/utils/port/sleep.h" +#include "td/utils/port/Stat.h" +#include "td/utils/port/thread.h" +#include "td/utils/Random.h" +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/tests.h" + +#include <atomic> +#include <clocale> +#include <limits> +#include <locale> + +using namespace td; + +#if TD_LINUX || TD_DARWIN +TEST(Misc, update_atime_saves_mtime) { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); + std::string name = "test_file"; + unlink(name).ignore(); + auto r_file = FileFd::open(name, FileFd::Read | FileFd::Flags::Create | FileFd::Flags::Truncate); + LOG_IF(ERROR, r_file.is_error()) << r_file.error(); + ASSERT_TRUE(r_file.is_ok()); + r_file.move_as_ok().close(); + + auto info = stat(name).ok(); + int32 tests_ok = 0; + int32 tests_wa = 0; + for (int i = 0; i < 10000; i++) { + update_atime(name).ensure(); + auto new_info = stat(name).ok(); + if (info.mtime_nsec_ == new_info.mtime_nsec_) { + tests_ok++; + } else { + tests_wa++; + info.mtime_nsec_ = new_info.mtime_nsec_; + } + ASSERT_EQ(info.mtime_nsec_, new_info.mtime_nsec_); + usleep_for(Random::fast(0, 1000)); + } + if (tests_wa > 0) { + LOG(ERROR) << "Access time was unexpectedly updated " << tests_wa << " times"; + } + unlink(name).ensure(); +} + +TEST(Misc, update_atime_change_atime) { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); + std::string name = "test_file"; + unlink(name).ignore(); + auto r_file = FileFd::open(name, FileFd::Read | FileFd::Flags::Create | FileFd::Flags::Truncate); + LOG_IF(ERROR, r_file.is_error()) << r_file.error(); + ASSERT_TRUE(r_file.is_ok()); + r_file.move_as_ok().close(); + auto info = stat(name).ok(); + // not enough for fat and e.t.c. + usleep_for(5000000); + update_atime(name).ensure(); + auto new_info = stat(name).ok(); + if (info.atime_nsec_ == new_info.atime_nsec_) { + LOG(ERROR) << "Access time was unexpectedly not changed"; + } + unlink(name).ensure(); +} +#endif + +TEST(Misc, errno_tls_bug) { + // That's a problem that should be avoided + // errno = 0; + // impl_.alloc(123); + // CHECK(errno == 0); + +#if !TD_THREAD_UNSUPPORTED && !TD_EVENTFD_UNSUPPORTED + EventFd test_event_fd; + test_event_fd.init(); + std::atomic<int> s(0); + s = 1; + td::thread th([&] { + while (s != 1) { + } + test_event_fd.acquire(); + }); + th.join(); + + for (int i = 0; i < 1000; i++) { + vector<EventFd> events(10); + vector<td::thread> threads; + for (auto &event : events) { + event.init(); + event.release(); + } + for (auto &event : events) { + threads.push_back(td::thread([&] { + { + EventFd tmp; + tmp.init(); + tmp.acquire(); + } + event.acquire(); + })); + } + for (auto &thread : threads) { + thread.join(); + } + } +#endif +} + +TEST(Misc, base64) { + ASSERT_TRUE(is_base64("dGVzdA==") == true); + ASSERT_TRUE(is_base64("dGVzdB==") == false); + ASSERT_TRUE(is_base64("dGVzdA=") == false); + ASSERT_TRUE(is_base64("dGVzdA") == false); + ASSERT_TRUE(is_base64("dGVz") == true); + ASSERT_TRUE(is_base64("") == true); + ASSERT_TRUE(is_base64("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/") == true); + ASSERT_TRUE(is_base64("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=") == false); + ASSERT_TRUE(is_base64("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-/") == false); + ASSERT_TRUE(is_base64("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_") == false); + ASSERT_TRUE(is_base64("====") == false); + + ASSERT_TRUE(is_base64url("dGVzdA==") == true); + ASSERT_TRUE(is_base64url("dGVzdB==") == false); + ASSERT_TRUE(is_base64url("dGVzdA=") == false); + ASSERT_TRUE(is_base64url("dGVzdA") == true); + ASSERT_TRUE(is_base64url("dGVz") == true); + ASSERT_TRUE(is_base64url("") == true); + ASSERT_TRUE(is_base64url("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_") == true); + ASSERT_TRUE(is_base64url("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_=") == false); + ASSERT_TRUE(is_base64url("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-/") == false); + ASSERT_TRUE(is_base64url("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/") == false); + ASSERT_TRUE(is_base64url("====") == false); + + for (int l = 0; l < 300000; l += l / 20 + l / 1000 * 500 + 1) { + for (int t = 0; t < 10; t++) { + string s = rand_string(std::numeric_limits<char>::min(), std::numeric_limits<char>::max(), l); + string encoded = base64url_encode(s); + auto decoded = base64url_decode(encoded); + ASSERT_TRUE(decoded.is_ok()); + ASSERT_TRUE(decoded.ok() == s); + + encoded = base64_encode(s); + decoded = base64_decode(encoded); + ASSERT_TRUE(decoded.is_ok()); + ASSERT_TRUE(decoded.ok() == s); + } + } + + ASSERT_TRUE(base64url_decode("dGVzdA").is_ok()); + ASSERT_TRUE(base64url_decode("dGVzdB").is_error()); + ASSERT_TRUE(base64_encode(base64url_decode("dGVzdA").ok()) == "dGVzdA=="); + ASSERT_TRUE(base64_encode("any carnal pleas") == "YW55IGNhcm5hbCBwbGVhcw=="); + ASSERT_TRUE(base64_encode("any carnal pleasu") == "YW55IGNhcm5hbCBwbGVhc3U="); + ASSERT_TRUE(base64_encode("any carnal pleasur") == "YW55IGNhcm5hbCBwbGVhc3Vy"); + ASSERT_TRUE(base64_encode(" /'.;.';≤.];,].',[.;/,.;/]/..;!@#!*(%?::;!%\";") == + "ICAgICAgLycuOy4nO+KJpC5dOyxdLicsWy47LywuOy9dLy4uOyFAIyEqKCU/" + "Ojo7ISUiOw=="); +} + +TEST(Misc, to_integer) { + ASSERT_EQ(to_integer<int32>("-1234567"), -1234567); + ASSERT_EQ(to_integer<int64>("-1234567"), -1234567); + ASSERT_EQ(to_integer<uint32>("-1234567"), 0u); + ASSERT_EQ(to_integer<int16>("-1234567"), 10617); + ASSERT_EQ(to_integer<uint16>("-1234567"), 0u); + ASSERT_EQ(to_integer<int16>("-1254567"), -9383); + ASSERT_EQ(to_integer<uint16>("1254567"), 9383u); + ASSERT_EQ(to_integer<int64>("-12345678910111213"), -12345678910111213); + ASSERT_EQ(to_integer<uint64>("12345678910111213"), 12345678910111213ull); + + ASSERT_EQ(to_integer_safe<int32>("-1234567").ok(), -1234567); + ASSERT_EQ(to_integer_safe<int64>("-1234567").ok(), -1234567); + ASSERT_TRUE(to_integer_safe<uint32>("-1234567").is_error()); + ASSERT_TRUE(to_integer_safe<int16>("-1234567").is_error()); + ASSERT_TRUE(to_integer_safe<uint16>("-1234567").is_error()); + ASSERT_TRUE(to_integer_safe<int16>("-1254567").is_error()); + ASSERT_TRUE(to_integer_safe<uint16>("1254567").is_error()); + ASSERT_EQ(to_integer_safe<int64>("-12345678910111213").ok(), -12345678910111213); + ASSERT_EQ(to_integer_safe<uint64>("12345678910111213").ok(), 12345678910111213ull); + ASSERT_TRUE(to_integer_safe<uint64>("-12345678910111213").is_error()); +} + +static void test_to_double_one(CSlice str, Slice expected, int precision = 6) { + auto result = PSTRING() << td::StringBuilder::FixedDouble(to_double(str), precision); + if (expected != result) { + LOG(ERROR) << "To double conversion failed: have " << str << ", expected " << expected << ", parsed " + << to_double(str) << ", got " << result; + } +} + +static void test_to_double() { + test_to_double_one("0", "0.000000"); + test_to_double_one("1", "1.000000"); + test_to_double_one("-10", "-10.000000"); + test_to_double_one("1.234", "1.234000"); + test_to_double_one("-1.234e2", "-123.400000"); + test_to_double_one("inf", "inf"); + test_to_double_one(" inF asdasd", "inf"); + test_to_double_one(" inFasdasd", "0.000000"); + test_to_double_one(" NaN", "nan"); + test_to_double_one(" 12345678910111213141516171819 asdasd", "12345678910111213670658736128.000000"); + test_to_double_one("1.234567891011121314E123", + "1234567891011121363209105003376291141757777526749278953577304234065881343284952489418916814035346" + "625663604561924259911303168.000000"); + test_to_double_one("1.234567891011121314E-9", "0.000000"); + test_to_double_one("123456789", "123456789.000000"); + test_to_double_one("-1,234567891011121314E123", "-1.000000"); + test_to_double_one("123456789", "123456789", 0); + test_to_double_one("1.23456789", "1", 0); + test_to_double_one("1.23456789", "1.2", 1); + test_to_double_one("1.23456789", "1.23", 2); + test_to_double_one("1.23456789", "1.235", 3); + test_to_double_one("1.23456789", "1.2346", 4); + test_to_double_one("1.23456789", "1.23457", 5); + test_to_double_one("1.23456789", "1.234568", 6); + test_to_double_one("1.23456789", "1.2345679", 7); + test_to_double_one("1.23456789", "1.23456789", 8); + test_to_double_one("1.23456789", "1.234567890", 9); + test_to_double_one("1.23456789", "1.2345678900", 10); +} + +TEST(Misc, to_double) { + test_to_double(); + const char *locale_name = (std::setlocale(LC_ALL, "fr-FR") == nullptr ? "" : "fr-FR"); + std::locale new_locale(locale_name); + std::locale::global(new_locale); + test_to_double(); + std::locale::global(std::locale::classic()); + test_to_double(); +} + +static void test_get_url_query_file_name_one(const char *prefix, const char *suffix, const char *file_name) { + auto path = string(prefix) + string(file_name) + string(suffix); + ASSERT_STREQ(file_name, get_url_query_file_name(path)); + ASSERT_STREQ(file_name, get_url_file_name("http://telegram.org" + path)); + ASSERT_STREQ(file_name, get_url_file_name("http://telegram.org:80" + path)); + ASSERT_STREQ(file_name, get_url_file_name("telegram.org" + path)); +} + +TEST(Misc, get_url_query_file_name) { + for (auto suffix : {"?t=1#test", "#test?t=1", "#?t=1", "?t=1#", "#test", "?t=1", "#", "?", ""}) { + test_get_url_query_file_name_one("", suffix, ""); + test_get_url_query_file_name_one("/", suffix, ""); + test_get_url_query_file_name_one("/a/adasd/", suffix, ""); + test_get_url_query_file_name_one("/a/lklrjetn/", suffix, "adasd.asdas"); + test_get_url_query_file_name_one("/", suffix, "a123asadas"); + test_get_url_query_file_name_one("/", suffix, "\\a\\1\\2\\3\\a\\s\\a\\das"); + } +} diff --git a/libs/tdlib/td/tdutils/test/pq.cpp b/libs/tdlib/td/tdutils/test/pq.cpp new file mode 100644 index 0000000000..5210cc2638 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/pq.cpp @@ -0,0 +1,118 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/tests.h" + +#include "td/utils/BigNum.h" +#include "td/utils/common.h" +#include "td/utils/crypto.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +#include <algorithm> +#include <limits> +#include <utility> + +REGISTER_TESTS(pq) + +using namespace td; + +#if TD_HAVE_OPENSSL +static bool is_prime(uint64 x) { + for (uint64 d = 2; d < x && d * d <= x; d++) { + if (x % d == 0) { + return false; + } + } + return true; +} + +static std::vector<uint64> gen_primes(uint64 L, uint64 R, int limit = 0) { + std::vector<uint64> res; + for (auto x = L; x <= R && (limit <= 0 || res.size() < static_cast<std::size_t>(limit)); x++) { + if (is_prime(x)) { + res.push_back(x); + } + } + return res; +} + +static std::vector<uint64> gen_primes() { + std::vector<uint64> result; + append(result, gen_primes(1, 100)); + append(result, gen_primes((1ull << 31) - 500000, std::numeric_limits<uint64>::max(), 5)); + append(result, gen_primes((1ull << 32) - 500000, std::numeric_limits<uint64>::max(), 5)); + append(result, gen_primes((1ull << 39) - 500000, std::numeric_limits<uint64>::max(), 1)); + return result; +} + +using PqQuery = std::pair<uint64, uint64>; +static bool cmp(const PqQuery &a, const PqQuery &b) { + return a.first * a.second < b.first * b.second; +} +static std::vector<PqQuery> gen_pq_queries() { + std::vector<PqQuery> res; + auto primes = gen_primes(); + for (auto q : primes) { + for (auto p : primes) { + if (p > q) { + break; + } + res.emplace_back(p, q); + } + } + std::sort(res.begin(), res.end(), cmp); + return res; +} + +static void test_pq(uint64 first, uint64 second) { + BigNum p = BigNum::from_decimal(PSLICE() << first); + BigNum q = BigNum::from_decimal(PSLICE() << second); + + BigNum pq; + BigNumContext context; + BigNum::mul(pq, p, q, context); + std::string pq_str = pq.to_binary(); + + std::string p_str, q_str; + int err = td::pq_factorize(pq_str, &p_str, &q_str); + CHECK(err == 0) << first << " * " << second; + + BigNum p_res = BigNum::from_binary(p_str); + BigNum q_res = BigNum::from_binary(q_str); + + CHECK(p_str == p.to_binary()) << td::tag("got", p_res.to_decimal()) << td::tag("expected", first); + CHECK(q_str == q.to_binary()) << td::tag("got", q_res.to_decimal()) << td::tag("expected", second); +} +#endif + +TEST(CryptoPQ, hands) { + ASSERT_EQ(1ull, td::pq_factorize(0)); + ASSERT_EQ(1ull, td::pq_factorize(1)); + ASSERT_EQ(1ull, td::pq_factorize(2)); + ASSERT_EQ(1ull, td::pq_factorize(3)); + ASSERT_EQ(2ull, td::pq_factorize(4)); + ASSERT_EQ(1ull, td::pq_factorize(5)); + ASSERT_EQ(3ull, td::pq_factorize(7 * 3)); + ASSERT_EQ(179424611ull, td::pq_factorize(179424611ull * 179424673ull)); + +#if TD_HAVE_OPENSSL + test_pq(4294467311, 4294467449); +#endif +} + +#if TD_HAVE_OPENSSL +TEST(CryptoPQ, generated_slow) { + for (int i = 0; i < 100000; i++) { + test_pq(2, 2); + } + auto queries = gen_pq_queries(); + for (auto query : queries) { + test_pq(query.first, query.second); + } +} +#endif
\ No newline at end of file diff --git a/libs/tdlib/td/tdutils/test/variant.cpp b/libs/tdlib/td/tdutils/test/variant.cpp new file mode 100644 index 0000000000..5c5e18d1d8 --- /dev/null +++ b/libs/tdlib/td/tdutils/test/variant.cpp @@ -0,0 +1,75 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/tests.h" +#include "td/utils/Variant.h" + +REGISTER_TESTS(variant); + +using namespace td; + +static const size_t BUF_SIZE = 1024 * 1024; +static char buf[BUF_SIZE], buf2[BUF_SIZE]; +static StringBuilder sb(MutableSlice(buf, BUF_SIZE - 1)); +static StringBuilder sb2(MutableSlice(buf2, BUF_SIZE - 1)); + +static std::string move_sb() { + auto res = sb.as_cslice().str(); + sb.clear(); + return res; +} + +static std::string name(int id) { + if (id == 1) { + return "A"; + } + if (id == 2) { + return "B"; + } + if (id == 3) { + return "C"; + } + return ""; +} + +template <int id> +class Class { + public: + Class() { + sb << "+" << name(id); + } + Class(const Class &) = delete; + Class &operator=(const Class &) = delete; + Class(Class &&) = delete; + Class &operator=(Class &&) = delete; + ~Class() { + sb << "-" << name(id); + } +}; + +using A = Class<1>; +using B = Class<2>; +using C = Class<3>; + +TEST(Variant, simple) { + { + Variant<std::unique_ptr<A>, std::unique_ptr<B>, std::unique_ptr<C>> abc; + ASSERT_STREQ("", sb.as_cslice()); + abc = std::make_unique<A>(); + ASSERT_STREQ("+A", sb.as_cslice()); + sb.clear(); + abc = std::make_unique<B>(); + ASSERT_STREQ("+B-A", sb.as_cslice()); + sb.clear(); + abc = std::make_unique<C>(); + ASSERT_STREQ("+C-B", sb.as_cslice()); + sb.clear(); + } + ASSERT_STREQ("-C", move_sb()); + sb.clear(); +}; |