diff options
author | aunsane <aunsane@gmail.com> | 2018-04-27 21:33:17 +0300 |
---|---|---|
committer | aunsane <aunsane@gmail.com> | 2018-04-27 21:33:17 +0300 |
commit | e1ec72eab6d00b3ba38e5932bc88920f103b6e4a (patch) | |
tree | 999de2725a83e30fbbf6576200525d4ef0c5fe38 /protocols/Telegram/tdlib/td/tdnet | |
parent | b9ce1d4d98525490ca1a38e2d9fd4f3369adb3e0 (diff) |
Telegram: initial commit
- tdlib moved to telegram dir
Diffstat (limited to 'protocols/Telegram/tdlib/td/tdnet')
29 files changed, 3141 insertions, 0 deletions
diff --git a/protocols/Telegram/tdlib/td/tdnet/CMakeLists.txt b/protocols/Telegram/tdlib/td/tdnet/CMakeLists.txt new file mode 100644 index 0000000000..823ed027d6 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/CMakeLists.txt @@ -0,0 +1,54 @@ +cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) + +if (NOT OPENSSL_FOUND) + find_package(OpenSSL REQUIRED) + find_package(ZLIB REQUIRED) +endif() + +#SOURCE SETS +set(TDNET_SOURCE + td/net/GetHostByNameActor.cpp + td/net/HttpChunkedByteFlow.cpp + td/net/HttpConnectionBase.cpp + td/net/HttpContentLengthByteFlow.cpp + td/net/HttpFile.cpp + td/net/HttpInboundConnection.cpp + td/net/HttpOutboundConnection.cpp + td/net/HttpQuery.cpp + td/net/HttpReader.cpp + td/net/Socks5.cpp + td/net/SslFd.cpp + td/net/TcpListener.cpp + td/net/Wget.cpp + + td/net/GetHostByNameActor.h + td/net/HttpChunkedByteFlow.h + td/net/HttpConnectionBase.h + td/net/HttpContentLengthByteFlow.h + td/net/HttpFile.h + td/net/HttpHeaderCreator.h + td/net/HttpInboundConnection.h + td/net/HttpOutboundConnection.h + td/net/HttpQuery.h + td/net/HttpReader.h + td/net/NetStats.h + td/net/Socks5.h + td/net/SslFd.h + td/net/TcpListener.h + td/net/Wget.h +) + +#RULES +#LIBRARIES + +add_library(tdnet STATIC ${TDNET_SOURCE}) +target_include_directories(tdnet PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>) +target_include_directories(tdnet SYSTEM PUBLIC $<BUILD_INTERFACE:${OPENSSL_INCLUDE_DIR}>) +target_link_libraries(tdnet PUBLIC tdutils tdactor ${OPENSSL_LIBRARIES} PRIVATE ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) + +install(TARGETS tdnet EXPORT TdTargets + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib + RUNTIME DESTINATION bin + INCLUDES DESTINATION include +) diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/GetHostByNameActor.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/GetHostByNameActor.cpp new file mode 100644 index 0000000000..b6cdcca0f0 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/GetHostByNameActor.cpp @@ -0,0 +1,48 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/GetHostByNameActor.h" + +#include "td/utils/logging.h" +#include "td/utils/Time.h" + +namespace td { +GetHostByNameActor::GetHostByNameActor(int32 ok_timeout, int32 error_timeout) + : ok_timeout_(ok_timeout), error_timeout_(error_timeout) { +} + +void GetHostByNameActor::run(std::string host, int port, td::Promise<td::IPAddress> promise) { + auto r_ip = load_ip(std::move(host), port); + promise.set_result(std::move(r_ip)); +} + +Result<td::IPAddress> GetHostByNameActor::load_ip(string host, int port) { + auto &value = cache_.emplace(host, Value{{}, 0}).first->second; + auto begin_time = td::Time::now(); + if (value.expire_at > begin_time) { + auto ip = value.ip.clone(); + if (ip.is_ok()) { + ip.ok_ref().set_port(port); + CHECK(ip.ok().get_port() == port); + } + return ip; + } + + td::IPAddress ip; + auto status = ip.init_host_port(host, port); + auto end_time = td::Time::now(); + LOG(WARNING) << "Init host = " << host << ", port = " << port << " in " << end_time - begin_time << " seconds to " + << ip; + + if (status.is_ok()) { + value = Value{ip, end_time + ok_timeout_}; + return ip; + } else { + value = Value{status.clone(), end_time + error_timeout_}; + return std::move(status); + } +} +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/GetHostByNameActor.h b/protocols/Telegram/tdlib/td/tdnet/td/net/GetHostByNameActor.h new file mode 100644 index 0000000000..b352a05d18 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/GetHostByNameActor.h @@ -0,0 +1,35 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/utils/port/IPAddress.h" +#include "td/utils/Status.h" + +#include <unordered_map> + +namespace td { +class GetHostByNameActor final : public td::Actor { + public: + explicit GetHostByNameActor(int32 ok_timeout = CACHE_TIME, int32 error_timeout = ERROR_CACHE_TIME); + void run(std::string host, int port, td::Promise<td::IPAddress> promise); + + private: + struct Value { + Result<td::IPAddress> ip; + double expire_at; + }; + std::unordered_map<string, Value> cache_; + static constexpr int32 CACHE_TIME = 60 * 29; // 29 minutes + static constexpr int32 ERROR_CACHE_TIME = 60 * 5; // 5 minutes + + int32 ok_timeout_; + int32 error_timeout_; + Result<td::IPAddress> load_ip(string host, int port) TD_WARN_UNUSED_RESULT; +}; +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpChunkedByteFlow.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpChunkedByteFlow.cpp new file mode 100644 index 0000000000..2edd225bfa --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpChunkedByteFlow.cpp @@ -0,0 +1,83 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpChunkedByteFlow.h" + +#include "td/utils/find_boundary.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Status.h" + +namespace td { + +void HttpChunkedByteFlow::loop() { + bool was_updated = false; + size_t need_size; + while (true) { + if (state_ == ReadChunkLength) { + bool ok = find_boundary(input_->clone(), "\r\n", len_); + if (len_ > 10) { + return finish(Status::Error(PSLICE() << "Too long length in chunked " + << input_->cut_head(len_).move_as_buffer_slice().as_slice())); + } + if (!ok) { + need_size = input_->size() + 1; + break; + } + auto s_len = input_->cut_head(len_).move_as_buffer_slice(); + input_->advance(2); + len_ = hex_to_integer<size_t>(s_len.as_slice()); + if (len_ > MAX_CHUNK_SIZE) { + return finish(Status::Error(PSLICE() << "Invalid chunk size " << tag("size", len_))); + } + save_len_ = len_; + state_ = ReadChunkContent; + } + + auto size = input_->size(); + auto ready = min(len_, size); + need_size = min(MIN_UPDATE_SIZE, len_ + 2); + if (size < need_size) { + break; + } + total_size_ += ready; + uncommited_size_ += ready; + if (total_size_ > MAX_SIZE) { + return finish(Status::Error(PSLICE() << "Too big query " << tag("size", input_->size()))); + } + + output_.append(input_->cut_head(ready)); + len_ -= ready; + if (uncommited_size_ >= MIN_UPDATE_SIZE) { + uncommited_size_ = 0; + was_updated = true; + } + + if (len_ == 0) { + if (input_->size() < 2) { + need_size = 2; + break; + } + input_->cut_head(2); + total_size_ += 2; + if (save_len_ == 0) { + return finish(Status::OK()); + } + state_ = ReadChunkLength; + len_ = 0; + } + } + if (was_updated) { + on_output_updated(); + } + if (!is_input_active_) { + return finish(Status::Error("Unexpected end of stream")); + } + set_need_size(need_size); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpChunkedByteFlow.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpChunkedByteFlow.h new file mode 100644 index 0000000000..9c62c3368e --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpChunkedByteFlow.h @@ -0,0 +1,28 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/ByteFlow.h" + +namespace td { + +class HttpChunkedByteFlow final : public ByteFlowBase { + public: + void loop() override; + + private: + static constexpr int MAX_CHUNK_SIZE = 15 << 20; // some reasonable limit + static constexpr int MAX_SIZE = 150 << 20; // some reasonable limit + static constexpr size_t MIN_UPDATE_SIZE = 1 << 14; + enum { ReadChunkLength, ReadChunkContent, OK } state_ = ReadChunkLength; + size_t len_ = 0; + size_t save_len_; + size_t total_size_ = 0; + size_t uncommited_size_ = 0; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpConnectionBase.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpConnectionBase.cpp new file mode 100644 index 0000000000..087ee5b790 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpConnectionBase.cpp @@ -0,0 +1,153 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpConnectionBase.h" + +#include "td/actor/actor.h" + +#include "td/net/HttpHeaderCreator.h" + +#include "td/utils/logging.h" +#include "td/utils/misc.h" + +namespace td { +namespace detail { + +HttpConnectionBase::HttpConnectionBase(State state, FdProxy fd, size_t max_post_size, size_t max_files, + int32 idle_timeout) + : state_(state) + , stream_connection_(std::move(fd)) + , max_post_size_(max_post_size) + , max_files_(max_files) + , idle_timeout_(idle_timeout) { + CHECK(state_ != State::Close); +} + +void HttpConnectionBase::live_event() { + if (idle_timeout_ != 0) { + set_timeout_in(idle_timeout_); + } +} + +void HttpConnectionBase::start_up() { + stream_connection_.get_fd().set_observer(this); + subscribe(stream_connection_.get_fd()); + reader_.init(&stream_connection_.input_buffer(), max_post_size_, max_files_); + if (state_ == State::Read) { + current_query_ = make_unique<HttpQuery>(); + } + live_event(); + yield(); +} +void HttpConnectionBase::tear_down() { + unsubscribe_before_close(stream_connection_.get_fd()); + stream_connection_.close(); +} + +void HttpConnectionBase::write_next(BufferSlice buffer) { + CHECK(state_ == State::Write); + stream_connection_.output_buffer().append(std::move(buffer)); + loop(); +} + +void HttpConnectionBase::write_ok() { + CHECK(state_ == State::Write); + current_query_ = make_unique<HttpQuery>(); + state_ = State::Read; + live_event(); + loop(); +} + +void HttpConnectionBase::write_error(Status error) { + CHECK(state_ == State::Write); + LOG(WARNING) << "Close http connection: " << error; + state_ = State::Close; + loop(); +} + +void HttpConnectionBase::timeout_expired() { + LOG(INFO) << "Idle timeout expired"; + + if (stream_connection_.need_flush_write()) { + on_error(Status::Error("Write timeout expired")); + } else if (state_ == State::Read) { + on_error(Status::Error("Read timeout expired")); + } + + stop(); +} +void HttpConnectionBase::loop() { + if (can_read(stream_connection_)) { + LOG(DEBUG) << "Can read from the connection"; + auto r = stream_connection_.flush_read(); + if (r.is_error()) { + if (!begins_with(r.error().message(), "SSL error {336134278")) { // if error is not yet outputed + LOG(INFO) << "flush_read error: " << r.error(); + } + on_error(Status::Error(r.error().public_message())); + return stop(); + } + } + + // TODO: read_next even when state_ == State::Write + + bool want_read = false; + if (state_ == State::Read) { + auto res = reader_.read_next(current_query_.get()); + if (res.is_error()) { + live_event(); + state_ = State::Write; + LOG(INFO) << res.error(); + HttpHeaderCreator hc; + hc.init_status_line(res.error().code()); + hc.set_content_size(0); + stream_connection_.output_buffer().append(hc.finish().ok()); + close_after_write_ = true; + on_error(Status::Error(res.error().public_message())); + } else if (res.ok() == 0) { + state_ = State::Write; + LOG(INFO) << "Send query to handler"; + live_event(); + on_query(std::move(current_query_)); + } else { + want_read = true; + } + } + + if (can_write(stream_connection_)) { + LOG(DEBUG) << "Can write to the connection"; + auto r = stream_connection_.flush_write(); + if (r.is_error()) { + LOG(INFO) << "flush_write error: " << r.error(); + on_error(Status::Error(r.error().public_message())); + } + if (close_after_write_ && !stream_connection_.need_flush_write()) { + return stop(); + } + } + + if (stream_connection_.get_fd().has_pending_error()) { + auto pending_error = stream_connection_.get_pending_error(); + LOG(INFO) << pending_error; + if (!close_after_write_) { + on_error(Status::Error(pending_error.public_message())); + } + state_ = State::Close; + } + if (can_close(stream_connection_)) { + LOG(INFO) << "Can close the connection"; + state_ = State::Close; + } + if (state_ == State::Close) { + LOG_IF(INFO, stream_connection_.need_flush_write()) << "Close nonempty connection"; + LOG_IF(INFO, want_read && + (stream_connection_.input_buffer().size() > 0 || current_query_->type_ != HttpQuery::Type::EMPTY)) + << "Close connection while reading request/response"; + return stop(); + } +} +} // namespace detail +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpConnectionBase.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpConnectionBase.h new file mode 100644 index 0000000000..1d420a3175 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpConnectionBase.h @@ -0,0 +1,164 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/net/HttpQuery.h" +#include "td/net/HttpReader.h" + +#include "td/utils/buffer.h" +#include "td/utils/BufferedFd.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +class FdInterface { + public: + FdInterface() = default; + FdInterface(const FdInterface &) = delete; + FdInterface &operator=(const FdInterface &) = delete; + FdInterface(FdInterface &&) = default; + FdInterface &operator=(FdInterface &&) = default; + virtual ~FdInterface() = default; + virtual const Fd &get_fd() const = 0; + virtual Fd &get_fd() = 0; + virtual int32 get_flags() const = 0; + virtual Status get_pending_error() TD_WARN_UNUSED_RESULT = 0; + + virtual Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT = 0; + virtual Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT = 0; + + virtual void close() = 0; + virtual bool empty() const = 0; +}; + +template <class FdT> +class FdToInterface : public FdInterface { + public: + FdToInterface() = default; + explicit FdToInterface(FdT fd) : fd_(std::move(fd)) { + } + + const Fd &get_fd() const final { + return fd_.get_fd(); + } + Fd &get_fd() final { + return fd_.get_fd(); + } + int32 get_flags() const final { + return fd_.get_flags(); + } + Status get_pending_error() final TD_WARN_UNUSED_RESULT { + return fd_.get_pending_error(); + } + + Result<size_t> write(Slice slice) final TD_WARN_UNUSED_RESULT { + return fd_.write(slice); + } + Result<size_t> read(MutableSlice slice) final TD_WARN_UNUSED_RESULT { + return fd_.read(slice); + } + + void close() final { + fd_.close(); + } + bool empty() const final { + return fd_.empty(); + } + + private: + FdT fd_; +}; + +template <class FdT> +std::unique_ptr<FdInterface> make_fd_interface(FdT fd) { + return make_unique<FdToInterface<FdT>>(std::move(fd)); +} + +class FdProxy { + public: + FdProxy() = default; + explicit FdProxy(std::unique_ptr<FdInterface> fd) : fd_(std::move(fd)) { + } + + const Fd &get_fd() const { + return fd_->get_fd(); + } + Fd &get_fd() { + return fd_->get_fd(); + } + int32 get_flags() const { + return fd_->get_flags(); + } + Status get_pending_error() TD_WARN_UNUSED_RESULT { + return fd_->get_pending_error(); + } + + Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT { + return fd_->write(slice); + } + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT { + return fd_->read(slice); + } + + void close() { + fd_->close(); + } + bool empty() const { + return fd_->empty(); + } + + private: + std::unique_ptr<FdInterface> fd_; +}; + +template <class FdT> +FdProxy make_fd_proxy(FdT fd) { + return FdProxy(make_fd_interface(std::move(fd))); +} + +namespace detail { +class HttpConnectionBase : public Actor { + public: + void write_next(BufferSlice buffer); + void write_ok(); + void write_error(Status error); + + protected: + enum class State { Read, Write, Close }; + template <class FdT> + HttpConnectionBase(State state, FdT fd, size_t max_post_size, size_t max_files, int32 idle_timeout) + : HttpConnectionBase(state, make_fd_proxy(std::move(fd)), max_post_size, max_files, idle_timeout) { + } + HttpConnectionBase(State state, FdProxy fd, size_t max_post_size, size_t max_files, int32 idle_timeout); + + private: + using StreamConnection = BufferedFd<FdProxy>; + State state_; + StreamConnection stream_connection_; + size_t max_post_size_; + size_t max_files_; + int32 idle_timeout_; + HttpReader reader_; + HttpQueryPtr current_query_; + bool close_after_write_ = false; + + void live_event(); + + void start_up() override; + void tear_down() override; + void timeout_expired() override; + void loop() override; + + virtual void on_query(HttpQueryPtr) = 0; + virtual void on_error(Status error) = 0; +}; +} // namespace detail +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpContentLengthByteFlow.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpContentLengthByteFlow.cpp new file mode 100644 index 0000000000..ea299b3993 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpContentLengthByteFlow.cpp @@ -0,0 +1,34 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpContentLengthByteFlow.h" + +#include "td/utils/Status.h" + +namespace td { + +void HttpContentLengthByteFlow::loop() { + auto ready_size = input_->size(); + if (ready_size > len_) { + ready_size = len_; + } + auto need_size = min(MIN_UPDATE_SIZE, len_); + if (ready_size < need_size) { + set_need_size(need_size); + return; + } + output_.append(input_->cut_head(ready_size)); + len_ -= ready_size; + if (len_ == 0) { + return finish(Status::OK()); + } + if (!is_input_active_) { + return finish(Status::Error("Unexpected end of stream")); + } + on_output_updated(); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpContentLengthByteFlow.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpContentLengthByteFlow.h new file mode 100644 index 0000000000..18f86abdb0 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpContentLengthByteFlow.h @@ -0,0 +1,25 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/ByteFlow.h" + +namespace td { + +class HttpContentLengthByteFlow final : public ByteFlowBase { + public: + HttpContentLengthByteFlow() = default; + explicit HttpContentLengthByteFlow(size_t len) : len_(len) { + } + void loop() override; + + private: + static constexpr size_t MIN_UPDATE_SIZE = 1 << 14; + size_t len_ = 0; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpFile.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpFile.cpp new file mode 100644 index 0000000000..b4f6e6d16b --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpFile.cpp @@ -0,0 +1,25 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpFile.h" + +#include "td/net/HttpReader.h" + +#include "td/utils/format.h" + +namespace td { + +HttpFile::~HttpFile() { + if (!temp_file_name.empty()) { + HttpReader::delete_temp_file(temp_file_name); + } +} + +StringBuilder &operator<<(StringBuilder &sb, const HttpFile &file) { + return sb << tag("name", file.name) << tag("size", file.size); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpFile.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpFile.h new file mode 100644 index 0000000000..6f35843060 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpFile.h @@ -0,0 +1,49 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/common.h" +#include "td/utils/StringBuilder.h" + +namespace td { + +class HttpFile { + public: + string field_name; + string name; + string content_type; + int64 size; + string temp_file_name; + + HttpFile(string field_name, string name, string content_type, int64 size, string temp_file_name) + : field_name(std::move(field_name)) + , name(std::move(name)) + , content_type(std::move(content_type)) + , size(size) + , temp_file_name(std::move(temp_file_name)) { + } + + HttpFile(const HttpFile &) = delete; + HttpFile &operator=(const HttpFile &) = delete; + + HttpFile(HttpFile &&other) + : field_name(std::move(other.field_name)) + , name(std::move(other.name)) + , content_type(std::move(other.content_type)) + , size(other.size) + , temp_file_name(std::move(other.temp_file_name)) { + other.temp_file_name.clear(); + } + + HttpFile &operator=(HttpFile &&) = delete; + + ~HttpFile(); +}; + +StringBuilder &operator<<(StringBuilder &sb, const HttpFile &file); + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpHeaderCreator.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpHeaderCreator.h new file mode 100644 index 0000000000..d3e84e5dbf --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpHeaderCreator.h @@ -0,0 +1,139 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/logging.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +namespace td { + +class HttpHeaderCreator { + public: + static constexpr size_t MAX_HEADER = 4096; + HttpHeaderCreator() : sb_({header_, MAX_HEADER}) { + } + void init_ok() { + sb_ = StringBuilder({header_, MAX_HEADER}); + sb_ << "HTTP/1.1 200 OK\r\n"; + } + void init_get(Slice url) { + sb_ = StringBuilder({header_, MAX_HEADER}); + sb_ << "GET " << url << " HTTP/1.1\r\n"; + } + void init_post(Slice url) { + sb_ = StringBuilder({header_, MAX_HEADER}); + sb_ << "POST " << url << " HTTP/1.1\r\n"; + } + void init_error(int code, Slice reason) { + sb_ = StringBuilder({header_, MAX_HEADER}); + sb_ << "HTTP/1.1 " << code << " " << reason << "\r\n"; + } + void init_status_line(int http_status_code) { + sb_ = StringBuilder({header_, MAX_HEADER}); + sb_ << "HTTP/1.1 " << http_status_code << " " << get_status_line(http_status_code) << "\r\n"; + } + void add_header(Slice key, Slice value) { + sb_ << key << ": " << value << "\r\n"; + } + void set_content_type(Slice type) { + add_header("Content-Type", type); + } + void set_content_size(size_t size) { + add_header("Content-Length", to_string(size)); + } + void set_keep_alive() { + add_header("Connection", "keep-alive"); + } + + Result<Slice> finish(Slice content = {}) TD_WARN_UNUSED_RESULT { + sb_ << "\r\n"; + if (!content.empty()) { + sb_ << content; + } + if (sb_.is_error()) { + return Status::Error("Too much headers"); + } + return sb_.as_cslice(); + } + + private: + static CSlice get_status_line(int http_status_code) { + if (http_status_code == 200) { + return CSlice("OK"); + } + switch (http_status_code) { + case 201: + return CSlice("Created"); + case 202: + return CSlice("Accepted"); + case 204: + return CSlice("No Content"); + case 206: + return CSlice("Partial Content"); + case 301: + return CSlice("Moved Permanently"); + case 302: + return CSlice("Found"); + case 303: + return CSlice("See Other"); + case 304: + return CSlice("Not Modified"); + case 307: + return CSlice("Temporary Redirect"); + case 400: + return CSlice("Bad Request"); + case 401: + return CSlice("Unauthorized"); + case 403: + return CSlice("Forbidden"); + case 404: + return CSlice("Not Found"); + case 405: + return CSlice("Method Not Allowed"); + case 406: + return CSlice("Not Acceptable"); + case 408: + return CSlice("Request Timeout"); + case 409: + return CSlice("Conflict"); + case 411: + return CSlice("Length Required"); + case 413: + return CSlice("Request Entity Too Large"); + case 414: + return CSlice("Request-URI Too Long"); + case 415: + return CSlice("Unsupported Media Type"); + case 418: + return CSlice("I'm a teapot"); + case 429: + return CSlice("Too Many Requests"); + case 431: + return CSlice("Request Header Fields Too Large"); + case 480: + return CSlice("Temporarily Unavailable"); + case 501: + return CSlice("Not Implemented"); + case 502: + return CSlice("Bad Gateway"); + case 503: + return CSlice("Service Unavailable"); + case 505: + return CSlice("HTTP Version Not Supported"); + default: + LOG_IF(ERROR, http_status_code != 500) << "Unsupported status code " << http_status_code << " returned"; + return CSlice("Internal Server Error"); + } + } + + char header_[MAX_HEADER]; + StringBuilder sb_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpInboundConnection.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpInboundConnection.cpp new file mode 100644 index 0000000000..533cdd5407 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpInboundConnection.cpp @@ -0,0 +1,28 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpInboundConnection.h" + +#include "td/utils/logging.h" + +namespace td { +// HttpInboundConnection implementation +HttpInboundConnection::HttpInboundConnection(SocketFd fd, size_t max_post_size, size_t max_files, int32 idle_timeout, + ActorShared<Callback> callback) + : HttpConnectionBase(State::Read, std::move(fd), max_post_size, max_files, idle_timeout) + , callback_(std::move(callback)) { +} + +void HttpInboundConnection::on_query(HttpQueryPtr query) { + CHECK(!callback_.empty()); + send_closure(callback_, &Callback::handle, std::move(query), ActorOwn<HttpInboundConnection>(actor_id(this))); +} + +void HttpInboundConnection::on_error(Status error) { + // nothing to do +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpInboundConnection.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpInboundConnection.h new file mode 100644 index 0000000000..013b024592 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpInboundConnection.h @@ -0,0 +1,43 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/net/HttpConnectionBase.h" +#include "td/net/HttpQuery.h" + +#include "td/utils/port/SocketFd.h" +#include "td/utils/Status.h" + +namespace td { + +class HttpInboundConnection final : public detail::HttpConnectionBase { + public: + class Callback : public Actor { + public: + virtual void handle(HttpQueryPtr query, ActorOwn<HttpInboundConnection> connection) = 0; + }; + // Inherited interface + // void write_next(BufferSlice buffer); + // void write_ok(); + // void write_error(Status error); + + HttpInboundConnection(SocketFd fd, size_t max_post_size, size_t max_files, int32 idle_timeout, + ActorShared<Callback> callback); + + private: + void on_query(HttpQueryPtr query) override; + void on_error(Status error) override; + void hangup() override { + callback_.release(); + stop(); + } + ActorShared<Callback> callback_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpOutboundConnection.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpOutboundConnection.cpp new file mode 100644 index 0000000000..f6efe7e07a --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpOutboundConnection.cpp @@ -0,0 +1,23 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpOutboundConnection.h" + +#include "td/utils/logging.h" + +namespace td { +// HttpOutboundConnection implementation +void HttpOutboundConnection::on_query(HttpQueryPtr query) { + CHECK(!callback_.empty()); + send_closure(callback_, &Callback::handle, std::move(query)); +} + +void HttpOutboundConnection::on_error(Status error) { + CHECK(!callback_.empty()); + send_closure(callback_, &Callback::on_connection_error, std::move(error)); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpOutboundConnection.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpOutboundConnection.h new file mode 100644 index 0000000000..d7496c59c4 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpOutboundConnection.h @@ -0,0 +1,46 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/net/HttpConnectionBase.h" +#include "td/net/HttpQuery.h" + +#include "td/utils/Status.h" + +namespace td { + +class HttpOutboundConnection final : public detail::HttpConnectionBase { + public: + class Callback : public Actor { + public: + virtual void handle(HttpQueryPtr query) = 0; + virtual void on_connection_error(Status error) = 0; // TODO rename to on_error + }; + template <class FdT> + HttpOutboundConnection(FdT fd, size_t max_post_size, size_t max_files, int32 idle_timeout, + ActorShared<Callback> callback) + : HttpConnectionBase(HttpConnectionBase::State::Write, std::move(fd), max_post_size, max_files, idle_timeout) + , callback_(std::move(callback)) { + } + // Inherited interface + // void write_next(BufferSlice buffer); + // void write_ok(); + // void write_error(Status error); + + private: + void on_query(HttpQueryPtr query) override; + void on_error(Status error) override; + void hangup() override { + callback_.release(); + HttpConnectionBase::hangup(); + } + ActorShared<Callback> callback_; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpQuery.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpQuery.cpp new file mode 100644 index 0000000000..b4af0eef3f --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpQuery.cpp @@ -0,0 +1,70 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpQuery.h" + +#include <algorithm> + +namespace td { + +Slice HttpQuery::header(Slice key) const { + auto it = std::find_if(headers_.begin(), headers_.end(), + [&key](const std::pair<MutableSlice, MutableSlice> &s) { return s.first == key; }); + return it == headers_.end() ? Slice() : it->second; +} + +MutableSlice HttpQuery::arg(Slice key) const { + auto it = std::find_if(args_.begin(), args_.end(), + [&key](const std::pair<MutableSlice, MutableSlice> &s) { return s.first == key; }); + return it == args_.end() ? MutableSlice() : it->second; +} + +std::vector<std::pair<string, string>> HttpQuery::string_args() const { + std::vector<std::pair<string, string>> res; + for (auto &it : args_) { + res.push_back(std::make_pair(it.first.str(), it.second.str())); + } + return res; +} + +StringBuilder &operator<<(StringBuilder &sb, const HttpQuery &q) { + switch (q.type_) { + case HttpQuery::Type::EMPTY: + sb << "EMPTY"; + return sb; + case HttpQuery::Type::GET: + sb << "GET"; + break; + case HttpQuery::Type::POST: + sb << "POST"; + break; + case HttpQuery::Type::RESPONSE: + sb << "RESPONSE"; + break; + } + if (q.type_ == HttpQuery::Type::RESPONSE) { + sb << ":" << q.code_ << ":" << q.reason_; + } else { + sb << ":" << q.url_path_; + for (auto &key_value : q.args_) { + sb << ":[" << key_value.first << ":" << key_value.second << "]"; + } + } + if (q.keep_alive_) { + sb << ":keep-alive"; + } + sb << "\n"; + for (auto &key_value : q.headers_) { + sb << key_value.first << "=" << key_value.second << "\n"; + } + sb << "BEGIN CONTENT\n"; + sb << q.content_; + sb << "END CONTENT\n"; + + return sb; +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpQuery.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpQuery.h new file mode 100644 index 0000000000..acab74ac66 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpQuery.h @@ -0,0 +1,47 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/net/HttpFile.h" + +#include "td/utils/buffer.h" +#include "td/utils/common.h" +#include "td/utils/Slice.h" +#include "td/utils/StringBuilder.h" + +#include <utility> + +namespace td { + +class HttpQuery { + public: + enum class Type : int8 { EMPTY, GET, POST, RESPONSE }; + + std::vector<BufferSlice> container_; + Type type_; + MutableSlice url_path_; + std::vector<std::pair<MutableSlice, MutableSlice>> args_; + int code_; + MutableSlice reason_; + + bool keep_alive_; + std::vector<std::pair<MutableSlice, MutableSlice>> headers_; + std::vector<HttpFile> files_; + MutableSlice content_; + + Slice header(Slice key) const; + + MutableSlice arg(Slice key) const; + + std::vector<std::pair<string, string>> string_args() const; +}; + +using HttpQueryPtr = std::unique_ptr<HttpQuery>; + +StringBuilder &operator<<(StringBuilder &sb, const HttpQuery &q); + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpReader.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpReader.cpp new file mode 100644 index 0000000000..1cfa7666a7 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpReader.cpp @@ -0,0 +1,814 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/HttpReader.h" + +#include "td/utils/filesystem.h" +#include "td/utils/find_boundary.h" +#include "td/utils/format.h" +#include "td/utils/Gzip.h" +#include "td/utils/HttpUrl.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/Parser.h" +#include "td/utils/PathView.h" +#include "td/utils/port/path.h" + +#include <cstring> + +namespace td { + +constexpr const char HttpReader::TEMP_DIRECTORY_PREFIX[]; + +static size_t urldecode(Slice from, MutableSlice to, bool decode_plus_sign_as_space) { + size_t to_i = 0; + CHECK(to.size() >= from.size()); + for (size_t from_i = 0, n = from.size(); from_i < n; from_i++) { + if (from[from_i] == '%' && from_i + 2 < n) { + int high = hex_to_int(from[from_i + 1]); + int low = hex_to_int(from[from_i + 2]); + if (high < 16 && low < 16) { + to[to_i++] = static_cast<char>(high * 16 + low); + from_i += 2; + continue; + } + } + to[to_i++] = decode_plus_sign_as_space && from[from_i] == '+' ? ' ' : from[from_i]; + } + return to_i; +} + +static MutableSlice urldecode_inplace(MutableSlice str, bool decode_plus_sign_as_space) { + size_t result_size = urldecode(str, str, decode_plus_sign_as_space); + str.truncate(result_size); + return str; +} + +void HttpReader::init(ChainBufferReader *input, size_t max_post_size, size_t max_files) { + input_ = input; + state_ = ReadHeaders; + headers_read_length_ = 0; + content_length_ = 0; + query_ = nullptr; + max_post_size_ = max_post_size; + max_files_ = max_files; + total_parameters_length_ = 0; + total_headers_length_ = 0; +} + +Result<size_t> HttpReader::read_next(HttpQuery *query) { + if (query_ != query) { + CHECK(query_ == nullptr); + query_ = query; + } + size_t need_size = input_->size() + 1; + while (true) { + if (state_ != ReadHeaders) { + flow_source_.wakeup(); + if (flow_sink_.is_ready() && flow_sink_.status().is_error()) { + return Status::Error(400, "Bad Request: " + flow_sink_.status().message().str()); + } + need_size = flow_source_.get_need_size(); + if (need_size == 0) { + need_size = input_->size() + 1; + } + } + switch (state_) { + case ReadHeaders: { + auto result = split_header(); + if (result.is_error() || result.ok() != 0) { + return result; + } + if (transfer_encoding_.empty() && content_length_ == 0) { + break; + } + + flow_source_ = ByteFlowSource(input_); + ByteFlowInterface *source = &flow_source_; + if (transfer_encoding_.empty()) { + content_length_flow_ = HttpContentLengthByteFlow(content_length_); + *source >> content_length_flow_; + source = &content_length_flow_; + } else if (transfer_encoding_ == "chunked") { + chunked_flow_ = HttpChunkedByteFlow(); + *source >> chunked_flow_; + source = &chunked_flow_; + } else { + LOG(ERROR) << "Unsupported " << tag("transfer-encoding", transfer_encoding_); + return Status::Error(501, "Unimplemented: unsupported transfer-encoding"); + } + + if (content_encoding_.empty()) { + } else if (content_encoding_ == "gzip" || content_encoding_ == "deflate") { + gzip_flow_ = GzipByteFlow(Gzip::Decode); + gzip_flow_.set_max_output_size(MAX_FILE_SIZE); + *source >> gzip_flow_; + source = &gzip_flow_; + } else { + LOG(ERROR) << "Unsupported " << tag("content-encoding", content_encoding_); + return Status::Error(415, "Unsupported Media Type: unsupported content-encoding"); + } + + flow_sink_ = ByteFlowSink(); + *source >> flow_sink_; + content_ = flow_sink_.get_output(); + + if (content_length_ > MAX_CONTENT_SIZE) { + return Status::Error(413, PSLICE() << "Request Entity Too Large: content length is " << content_length_); + } + + if (std::strstr(content_type_lowercased_.c_str(), "multipart/form-data")) { + state_ = ReadMultipartFormData; + + const char *p = std::strstr(content_type_lowercased_.c_str(), "boundary"); + if (p == nullptr) { + return Status::Error(400, "Bad Request: boundary not found"); + } + p += 8; + ptrdiff_t offset = p - content_type_lowercased_.c_str(); + p = static_cast<const char *>( + std::memchr(content_type_.begin() + offset, '=', content_type_.size() - offset)); + if (p == nullptr) { + return Status::Error(400, "Bad Request: boundary value not found"); + } + p++; + const char *end_p = static_cast<const char *>(std::memchr(p, ';', content_type_.end() - p)); + if (end_p == nullptr) { + end_p = content_type_.end(); + } + if (*p == '"' && p + 1 < end_p && end_p[-1] == '"') { + p++; + end_p--; + } + + Slice boundary(p, static_cast<size_t>(end_p - p)); + if (boundary.empty() || boundary.size() > MAX_BOUNDARY_LENGTH) { + return Status::Error(400, "Bad Request: boundary too big or empty"); + } + + boundary_ = "\r\n--" + boundary.str(); + form_data_parse_state_ = SkipPrologue; + form_data_read_length_ = 0; + form_data_skipped_length_ = 0; + } else if (std::strstr(content_type_lowercased_.c_str(), "application/x-www-form-urlencoded") || + std::strstr(content_type_lowercased_.c_str(), "application/json")) { + state_ = ReadArgs; + } else { + form_data_skipped_length_ = 0; + state_ = ReadContent; + } + continue; + } + case ReadContent: { + if (content_->size() > max_post_size_) { + state_ = ReadContentToFile; + continue; + } + if (flow_sink_.is_ready()) { + CHECK(query_->container_.size() == 1u); + query_->container_.emplace_back(content_->cut_head(content_->size()).move_as_buffer_slice()); + query_->content_ = query_->container_.back().as_slice(); + break; + } + + return need_size; + } + case ReadContentToFile: { + // save content to a file + if (temp_file_.empty()) { + auto file = open_temp_file("file"); + if (file.is_error()) { + return Status::Error(500, "Internal Server Error: can't create temporary file"); + } + } + + auto size = content_->size(); + if (size) { + TRY_STATUS(save_file_part(content_->cut_head(size).move_as_buffer_slice())); + } + if (flow_sink_.is_ready()) { + query_->files_.emplace_back("file", "", content_type_.str(), file_size_, temp_file_name_); + close_temp_file(); + break; + } + + return need_size; + } + case ReadArgs: { + auto size = content_->size(); + if (size > MAX_TOTAL_PARAMETERS_LENGTH - total_parameters_length_) { + return Status::Error(413, "Request Entity Too Large: too much parameters"); + } + + if (flow_sink_.is_ready()) { + query_->container_.emplace_back(content_->cut_head(size).move_as_buffer_slice()); + Status result; + if (std::strstr(content_type_lowercased_.c_str(), "application/x-www-form-urlencoded")) { + result = parse_parameters(query_->container_.back().as_slice()); + } else { + result = parse_json_parameters(query_->container_.back().as_slice()); + } + if (result.is_error()) { + if (result.code() == 413) { + return std::move(result); + } + LOG(INFO) << result.message(); + } + query_->content_ = MutableSlice(); + break; + } + + return need_size; + } + case ReadMultipartFormData: { + TRY_RESULT(result, parse_multipart_form_data()); + if (result) { + break; + } + return need_size; + } + default: + UNREACHABLE(); + } + break; + } + + init(input_, max_post_size_, max_files_); + return 0; +} + +// returns Status on wrong request +// returns true if parsing has finished +// returns false if need more data +Result<bool> HttpReader::parse_multipart_form_data() { + while (true) { + LOG(DEBUG) << "Parsing multipart form data in state " << form_data_parse_state_; + switch (form_data_parse_state_) { + case SkipPrologue: + if (find_boundary(content_->clone(), {boundary_.c_str() + 2, boundary_.size() - 2}, form_data_read_length_)) { + size_t to_skip = form_data_read_length_ + (boundary_.size() - 2); + content_->advance(to_skip); + form_data_skipped_length_ += to_skip; + form_data_read_length_ = 0; + + form_data_parse_state_ = ReadPartHeaders; + continue; + } + + content_->advance(form_data_read_length_); + form_data_skipped_length_ += form_data_read_length_; + form_data_read_length_ = 0; + return false; + case ReadPartHeaders: + if (find_boundary(content_->clone(), "\r\n\r\n", form_data_read_length_)) { + total_headers_length_ += form_data_read_length_; + if (total_headers_length_ > MAX_TOTAL_HEADERS_LENGTH) { + return Status::Error(431, "Request Header Fields Too Large: total headers size exceeded"); + } + if (form_data_read_length_ == 0) { + // there is no headers at all + return Status::Error(400, "Bad Request: headers in multipart/form-data are empty"); + } + + content_->advance(2); // "\r\n" after boundary + auto headers = content_->cut_head(form_data_read_length_).move_as_buffer_slice(); + CHECK(headers.as_slice().size() == form_data_read_length_); + LOG(DEBUG) << "Parse headers in multipart form data: \"" << headers.as_slice() << "\""; + content_->advance(2); + + form_data_skipped_length_ += form_data_read_length_ + 4; + form_data_read_length_ = 0; + + field_name_ = MutableSlice(); + file_field_name_.clear(); + field_content_type_ = "application/octet-stream"; + file_name_.clear(); + CHECK(temp_file_.empty()); + temp_file_name_.clear(); + + Parser headers_parser(headers.as_slice()); + while (headers_parser.status().is_ok() && !headers_parser.data().empty()) { + MutableSlice header_name = headers_parser.read_till(':'); + headers_parser.skip(':'); + char *header_value_begin = headers_parser.ptr(); + char *header_value_end; + do { + headers_parser.read_till('\r'); + header_value_end = headers_parser.ptr(); + headers_parser.skip('\r'); + headers_parser.skip('\n'); + } while (headers_parser.status().is_ok() && + (headers_parser.peek_char() == ' ' || headers_parser.peek_char() == '\t')); + + MutableSlice header_value(header_value_begin, header_value_end); + + header_name = trim(header_name); + header_value = trim(header_value); + to_lower_inplace(header_name); + + if (header_name == "content-disposition") { + if (header_value.substr(0, 10) != "form-data;") { + return Status::Error(400, "Bad Request: expected form-data content disposition"); + } + header_value.remove_prefix(10); + while (true) { + header_value = trim(header_value); + const char *key_end = + static_cast<const char *>(std::memchr(header_value.data(), '=', header_value.size())); + if (key_end == nullptr) { + break; + } + size_t key_size = key_end - header_value.data(); + auto key = header_value.substr(0, key_size); + key = trim(key); + + header_value.remove_prefix(key_size + 1); + const char *value_end = + static_cast<const char *>(std::memchr(header_value.data(), ';', header_value.size())); + size_t value_size; + if (value_end == nullptr) { + value_size = header_value.size(); + } else { + value_size = value_end - header_value.data(); + } + auto value = header_value.substr(0, value_size); + value = trim(value); + if (value.size() > 1u && value[0] == '"' && value.back() == '"') { + value = {value.data() + 1, value.size() - 2}; + } + header_value.remove_prefix(value_size + (header_value.size() > value_size)); + + if (key == "name") { + field_name_ = value; + } else if (key == "filename") { + file_name_ = value.str(); + } else { + // ignore unknown parts of header + } + } + } else if (header_name == "content-type") { + field_content_type_ = header_value.str(); + } else { + // ignore unknown header + } + } + + if (headers_parser.status().is_error()) { + return Status::Error(400, "Bad Request: can't parse form data headers"); + } + + if (field_name_.empty()) { + return Status::Error(400, "Bad Request: field name in multipart/form-data not found"); + } + + if (!file_name_.empty()) { + // file + if (query_->files_.size() == max_files_) { + return Status::Error(413, "Request Entity Too Large: too much files attached"); + } + auto file = open_temp_file(file_name_); + if (file.is_error()) { + return Status::Error(500, "Internal Server Error: can't create temporary file"); + } + + // don't need to save headers for files + file_field_name_ = field_name_.str(); + form_data_parse_state_ = ReadFile; + } else { + // save headers for query parameters. They contain header names + query_->container_.push_back(std::move(headers)); + form_data_parse_state_ = ReadPartValue; + } + + continue; + } + + if (total_headers_length_ + form_data_read_length_ > MAX_TOTAL_HEADERS_LENGTH) { + return Status::Error(431, "Request Header Fields Too Large: total headers size exceeded"); + } + return false; + case ReadPartValue: + if (find_boundary(content_->clone(), boundary_, form_data_read_length_)) { + if (total_parameters_length_ + form_data_read_length_ > MAX_TOTAL_PARAMETERS_LENGTH) { + return Status::Error(413, "Request Entity Too Large: too much parameters in form data"); + } + + query_->container_.emplace_back(content_->cut_head(form_data_read_length_).move_as_buffer_slice()); + MutableSlice value = query_->container_.back().as_slice(); + content_->advance(boundary_.size()); + form_data_skipped_length_ += form_data_read_length_ + boundary_.size(); + form_data_read_length_ = 0; + + if (begins_with(field_content_type_, "application/x-www-form-urlencoded")) { + // treat value as ordinary parameters + auto result = parse_parameters(value); + if (result.is_error()) { + return std::move(result); + } + } else { + total_parameters_length_ += form_data_read_length_; + LOG(DEBUG) << "Get ordinary parameter in multipart form data: \"" << field_name_ << "\": \"" << value + << "\""; + query_->args_.emplace_back(field_name_, value); + } + + form_data_parse_state_ = CheckForLastBoundary; + continue; + } + CHECK(content_->size() < form_data_read_length_ + boundary_.size()); + + if (total_parameters_length_ + form_data_read_length_ > MAX_TOTAL_PARAMETERS_LENGTH) { + return Status::Error(413, "Request Entity Too Large: too much parameters in form data"); + } + return false; + case ReadFile: { + if (find_boundary(content_->clone(), boundary_, form_data_read_length_)) { + auto file_part = content_->cut_head(form_data_read_length_).move_as_buffer_slice(); + content_->advance(boundary_.size()); + form_data_skipped_length_ += form_data_read_length_ + boundary_.size(); + form_data_read_length_ = 0; + + TRY_STATUS(save_file_part(std::move(file_part))); + + query_->files_.emplace_back(file_field_name_, file_name_, field_content_type_, file_size_, temp_file_name_); + close_temp_file(); + + form_data_parse_state_ = CheckForLastBoundary; + continue; + } + + // TODO optimize? + auto file_part = content_->cut_head(form_data_read_length_).move_as_buffer_slice(); + form_data_skipped_length_ += form_data_read_length_; + form_data_read_length_ = 0; + CHECK(content_->size() < boundary_.size()); + + TRY_STATUS(save_file_part(std::move(file_part))); + return false; + } + case CheckForLastBoundary: { + if (content_->size() < 2) { + // need more data + return false; + } + + auto range = content_->clone(); + char x[2]; + range.advance(2, {x, 2}); + if (x[0] == '-' && x[1] == '-') { + content_->advance(2); + form_data_skipped_length_ += 2; + form_data_parse_state_ = SkipEpilogue; + } else { + form_data_parse_state_ = ReadPartHeaders; + } + continue; + } + case SkipEpilogue: { + size_t size = content_->size(); + LOG(DEBUG) << "Skipping epilogue. Have " << size << " bytes"; + content_->advance(size); + form_data_skipped_length_ += size; + // TODO(now): check if form_data_skipped_length is too big + return flow_sink_.is_ready(); + } + default: + UNREACHABLE(); + } + break; + } + + return true; +} + +Result<size_t> HttpReader::split_header() { + if (find_boundary(input_->clone(), "\r\n\r\n", headers_read_length_)) { + query_->container_.clear(); + auto a = input_->cut_head(headers_read_length_ + 2); + auto b = a.move_as_buffer_slice(); + query_->container_.emplace_back(std::move(b)); + // query_->container_.emplace_back(input_->cut_head(headers_read_length_ + 2).move_as_buffer_slice()); + CHECK(query_->container_.back().size() == headers_read_length_ + 2); + input_->advance(2); + total_headers_length_ = headers_read_length_; + auto status = parse_head(query_->container_.back().as_slice()); + if (status.is_error()) { + return std::move(status); + } + return 0; + } + + if (input_->size() > MAX_TOTAL_HEADERS_LENGTH) { + return Status::Error(431, "Request Header Fields Too Large: total headers size exceeded"); + } + return input_->size() + 1; +} + +void HttpReader::process_header(MutableSlice header_name, MutableSlice header_value) { + header_name = trim(header_name); + header_value = trim(header_value); // TODO need to remove "\r\n" from value + to_lower_inplace(header_name); + LOG(DEBUG) << "process_header [" << header_name << "=>" << header_value << "]"; + query_->headers_.emplace_back(header_name, header_value); + // TODO: check if protocol is HTTP/1.1 + query_->keep_alive_ = true; + if (header_name == "content-length") { + content_length_ = to_integer<size_t>(header_value); + } else if (header_name == "connection") { + to_lower_inplace(header_value); + if (header_value == "close") { + query_->keep_alive_ = false; + } + } else if (header_name == "content-type") { + content_type_ = header_value; + content_type_lowercased_ = header_value.str(); + to_lower_inplace(content_type_lowercased_); + } else if (header_name == "content-encoding") { + to_lower_inplace(header_value); + content_encoding_ = header_value; + } else if (header_name == "transfer-encoding") { + to_lower_inplace(header_value); + transfer_encoding_ = header_value; + } +} + +Status HttpReader::parse_url(MutableSlice url) { + size_t url_path_size = 0; + while (url_path_size < url.size() && url[url_path_size] != '?' && url[url_path_size] != '#') { + url_path_size++; + } + + query_->url_path_ = urldecode_inplace({url.data(), url_path_size}, false); + + if (url_path_size == url.size() || url[url_path_size] != '?') { + return Status::OK(); + } + return parse_parameters(url.substr(url_path_size + 1)); +} + +Status HttpReader::parse_parameters(MutableSlice parameters) { + total_parameters_length_ += parameters.size(); + if (total_parameters_length_ > MAX_TOTAL_PARAMETERS_LENGTH) { + return Status::Error(413, "Request Entity Too Large: too much parameters"); + } + LOG(DEBUG) << "Parse parameters: \"" << parameters << "\""; + + Parser parser(parameters); + while (!parser.data().empty()) { + auto key_value = parser.read_till_nofail('&'); + parser.skip_nofail('&'); + Parser kv_parser(key_value); + auto key = urldecode_inplace(kv_parser.read_till_nofail('='), true); + kv_parser.skip_nofail('='); + auto value = urldecode_inplace(kv_parser.data(), true); + query_->args_.emplace_back(key, value); + } + + CHECK(parser.status().is_ok()); + return Status::OK(); +} + +Status HttpReader::parse_json_parameters(MutableSlice parameters) { + if (parameters.empty()) { + return Status::OK(); + } + + total_parameters_length_ += parameters.size(); + if (total_parameters_length_ > MAX_TOTAL_PARAMETERS_LENGTH) { + return Status::Error(413, "Request Entity Too Large: too much parameters"); + } + LOG(DEBUG) << "Parse json parameters: \"" << parameters << "\""; + + Parser parser(parameters); + parser.skip_whitespaces(); + parser.skip('{'); + if (parser.status().is_error()) { + return Status::Error(400, "Bad Request: json object expected"); + } + while (true) { + parser.skip_whitespaces(); + if (parser.try_skip('}')) { + parser.skip_whitespaces(); + if (parser.empty()) { + return Status::OK(); + } + return Status::Error(400, "Bad Request: unexpected data after object end"); + } + if (parser.empty()) { + return Status::Error(400, "Bad Request: expected parameter name"); + } + auto r_key = json_string_decode(parser); + if (r_key.is_error()) { + return Status::Error(400, string("Bad Request: can't parse parameter name: ") + r_key.error().message().c_str()); + } + parser.skip_whitespaces(); + if (!parser.try_skip(':')) { + return Status::Error(400, "Bad Request: can't parse object, ':' expected"); + } + parser.skip_whitespaces(); + Result<MutableSlice> r_value; + if (parser.peek_char() == '"') { + r_value = json_string_decode(parser); + } else { + const int32 DEFAULT_MAX_DEPTH = 100; + auto begin = parser.ptr(); + auto result = do_json_skip(parser, DEFAULT_MAX_DEPTH); + if (result.is_ok()) { + r_value = MutableSlice(begin, parser.ptr()); + } else { + r_value = result.move_as_error(); + } + } + if (r_value.is_error()) { + return Status::Error(400, + string("Bad Request: can't parse parameter value: ") + r_value.error().message().c_str()); + } + query_->args_.emplace_back(r_key.move_as_ok(), r_value.move_as_ok()); + + parser.skip_whitespaces(); + if (parser.peek_char() != '}' && !parser.try_skip(',')) { + return Status::Error(400, "Bad Request: expected next field or object end"); + } + } + UNREACHABLE(); + return Status::OK(); +} + +Status HttpReader::parse_head(MutableSlice head) { + Parser parser(head); + + Slice type = parser.read_till(' '); + parser.skip(' '); + // GET POST HTTP/1.1 + if (type == "GET") { + query_->type_ = HttpQuery::Type::GET; + } else if (type == "POST") { + query_->type_ = HttpQuery::Type::POST; + } else if (type.size() >= 4 && type.substr(0, 4) == "HTTP") { + if (type == "HTTP/1.1" || type == "HTTP/1.0") { + query_->type_ = HttpQuery::Type::RESPONSE; + } else { + LOG(INFO) << "Unsupported HTTP version: " << type; + return Status::Error(505, "HTTP Version Not Supported"); + } + } else { + LOG(INFO) << "Not Implemented " << tag("type", type) << tag("head", head); + return Status::Error(501, "Not Implemented"); + } + + query_->args_.clear(); + + if (query_->type_ == HttpQuery::Type::RESPONSE) { + query_->code_ = to_integer<int32>(parser.read_till(' ')); + parser.skip(' '); + query_->reason_ = parser.read_till('\r'); + } else { + auto url_version = parser.read_till('\r'); + auto space_pos = url_version.rfind(' '); + if (space_pos == static_cast<size_t>(-1)) { + return Status::Error(400, "Bad Request: wrong request line"); + } + + TRY_STATUS(parse_url(url_version.substr(0, space_pos))); + + auto http_version = url_version.substr(space_pos + 1); + if (http_version != "HTTP/1.1" && http_version != "HTTP/1.0") { + LOG(WARNING) << "Unsupported HTTP version: " << http_version; + return Status::Error(505, "HTTP Version Not Supported"); + } + } + parser.skip('\r'); + parser.skip('\n'); + + content_length_ = 0; + content_type_ = "application/octet-stream"; + content_type_lowercased_ = content_type_.str(); + transfer_encoding_ = ""; + content_encoding_ = ""; + + query_->keep_alive_ = false; + query_->headers_.clear(); + query_->files_.clear(); + query_->content_ = MutableSlice(); + while (parser.status().is_ok() && !parser.data().empty()) { + MutableSlice header_name = parser.read_till(':'); + parser.skip(':'); + char *header_value_begin = parser.ptr(); + char *header_value_end; + do { + parser.read_till('\r'); + header_value_end = parser.ptr(); + parser.skip('\r'); + parser.skip('\n'); + } while (parser.status().is_ok() && (parser.peek_char() == ' ' || parser.peek_char() == '\t')); + + process_header(header_name, {header_value_begin, header_value_end}); + } + return parser.status().is_ok() ? Status::OK() : Status::Error(400, "Bad Request"); +} + +Status HttpReader::open_temp_file(CSlice desired_file_name) { + CHECK(temp_file_.empty()); + + auto tmp_dir = get_temporary_dir(); + if (tmp_dir.empty()) { + return Status::Error("Can't find temporary directory"); + } + + TRY_RESULT(dir, realpath(tmp_dir, true)); + CHECK(!dir.empty()); + + auto first_try = try_open_temp_file(dir, desired_file_name); + if (first_try.is_ok()) { + return Status::OK(); + } + + // Creation of new file with desired name has failed. Trying to create unique directory for it + TRY_RESULT(directory, mkdtemp(dir, TEMP_DIRECTORY_PREFIX)); + auto second_try = try_open_temp_file(directory, desired_file_name); + if (second_try.is_ok()) { + return Status::OK(); + } + auto third_try = try_open_temp_file(directory, "file"); + if (third_try.is_ok()) { + return Status::OK(); + } + + rmdir(directory).ignore(); + LOG(WARNING) << "Failed to create temporary file " << desired_file_name << ": " << second_try.error(); + return second_try.move_as_error(); +} + +Status HttpReader::try_open_temp_file(Slice directory_name, CSlice desired_file_name) { + CHECK(temp_file_.empty()); + CHECK(!directory_name.empty()); + + string file_name = clean_filename(desired_file_name); + if (file_name.empty()) { + file_name = "file"; + } + + temp_file_name_.clear(); + temp_file_name_.reserve(directory_name.size() + 1 + file_name.size()); + temp_file_name_.append(directory_name.data(), directory_name.size()); + if (temp_file_name_.back() != TD_DIR_SLASH) { + temp_file_name_ += TD_DIR_SLASH; + } + temp_file_name_.append(file_name.data(), file_name.size()); + + TRY_RESULT(opened_file, FileFd::open(temp_file_name_, FileFd::Write | FileFd::CreateNew, 0640)); + + file_size_ = 0; + temp_file_ = std::move(opened_file); + LOG(DEBUG) << "Created temporary file " << temp_file_name_; + return Status::OK(); +} + +Status HttpReader::save_file_part(BufferSlice &&file_part) { + file_size_ += narrow_cast<int64>(file_part.size()); + if (file_size_ > MAX_FILE_SIZE) { + string file_name = temp_file_name_; + close_temp_file(); + delete_temp_file(file_name); + return Status::Error( + 413, PSLICE() << "Request Entity Too Large: file is too big to be uploaded " << tag("size", file_size_)); + } + + LOG(DEBUG) << "Save file part of size " << file_part.size() << " to file " << temp_file_name_; + auto result_written = temp_file_.write(file_part.as_slice()); + if (result_written.is_error() || result_written.ok() != file_part.size()) { + string file_name = temp_file_name_; + close_temp_file(); + delete_temp_file(file_name); + return Status::Error(500, "Internal server error: can't upload the file"); + } + return Status::OK(); +} + +void HttpReader::close_temp_file() { + LOG(DEBUG) << "Close temporary file " << temp_file_name_; + CHECK(!temp_file_.empty()); + temp_file_.close(); + CHECK(temp_file_.empty()); + temp_file_name_.clear(); +} + +void HttpReader::delete_temp_file(CSlice file_name) { + CHECK(!file_name.empty()); + LOG(DEBUG) << "Unlink temporary file " << file_name; + unlink(file_name).ignore(); + PathView path_view(file_name); + Slice parent = path_view.parent_dir(); + const size_t prefix_length = std::strlen(TEMP_DIRECTORY_PREFIX); + if (parent.size() >= prefix_length + 7 && + parent.substr(parent.size() - prefix_length - 7, prefix_length) == TEMP_DIRECTORY_PREFIX) { + LOG(DEBUG) << "Unlink temporary directory " << parent; + rmdir(Slice(parent.data(), parent.size() - 1).str()).ignore(); + } +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/HttpReader.h b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpReader.h new file mode 100644 index 0000000000..74067d1291 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/HttpReader.h @@ -0,0 +1,108 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/net/HttpChunkedByteFlow.h" +#include "td/net/HttpContentLengthByteFlow.h" +#include "td/net/HttpQuery.h" + +#include "td/utils/buffer.h" +#include "td/utils/ByteFlow.h" +#include "td/utils/common.h" +#include "td/utils/GzipByteFlow.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" +#include "td/utils/StringBuilder.h" + +#include <limits> + +namespace td { + +class HttpReader { + public: + void init(ChainBufferReader *input, size_t max_post_size = std::numeric_limits<size_t>::max(), + size_t max_files = 100); + Result<size_t> read_next(HttpQuery *query) TD_WARN_UNUSED_RESULT; // TODO move query to init + + HttpReader() = default; + HttpReader(const HttpReader &other) = delete; + HttpReader &operator=(const HttpReader &other) = delete; + HttpReader(HttpReader &&other) = delete; + HttpReader &operator=(HttpReader &&other) = delete; + ~HttpReader() { + if (!temp_file_.empty()) { + temp_file_.close(); + } + } + + static void delete_temp_file(CSlice file_name); + + private: + size_t max_post_size_; + size_t max_files_; + + enum { ReadHeaders, ReadContent, ReadContentToFile, ReadArgs, ReadMultipartFormData } state_; + size_t headers_read_length_; + size_t content_length_; + ChainBufferReader *input_; + ByteFlowSource flow_source_; + HttpChunkedByteFlow chunked_flow_; + GzipByteFlow gzip_flow_; + HttpContentLengthByteFlow content_length_flow_; + ByteFlowSink flow_sink_; + ChainBufferReader *content_; + + HttpQuery *query_; + Slice transfer_encoding_; + Slice content_encoding_; + Slice content_type_; + string content_type_lowercased_; + size_t total_parameters_length_; + size_t total_headers_length_; + + string boundary_; + size_t form_data_read_length_; + size_t form_data_skipped_length_; + enum { + SkipPrologue, + ReadPartHeaders, + ReadPartValue, + ReadFile, + CheckForLastBoundary, + SkipEpilogue + } form_data_parse_state_; + MutableSlice field_name_; + string file_field_name_; + string field_content_type_; + string file_name_; + FileFd temp_file_; + string temp_file_name_; + int64 file_size_; + + Result<size_t> split_header() TD_WARN_UNUSED_RESULT; + void process_header(MutableSlice header_name, MutableSlice header_value); + Result<bool> parse_multipart_form_data() TD_WARN_UNUSED_RESULT; + Status parse_url(MutableSlice url) TD_WARN_UNUSED_RESULT; + Status parse_parameters(MutableSlice parameters) TD_WARN_UNUSED_RESULT; + Status parse_json_parameters(MutableSlice parameters) TD_WARN_UNUSED_RESULT; + Status parse_head(MutableSlice head) TD_WARN_UNUSED_RESULT; + + Status open_temp_file(CSlice desired_file_name) TD_WARN_UNUSED_RESULT; + Status try_open_temp_file(Slice directory_name, CSlice desired_file_name) TD_WARN_UNUSED_RESULT; + Status save_file_part(BufferSlice &&file_part) TD_WARN_UNUSED_RESULT; + void close_temp_file(); + + static constexpr size_t MAX_CONTENT_SIZE = 150 << 20; // Some reasonable limit + static constexpr size_t MAX_TOTAL_PARAMETERS_LENGTH = 1 << 16; // Some reasonable limit + static constexpr size_t MAX_TOTAL_HEADERS_LENGTH = 1 << 18; // Some reasonable limit + static constexpr size_t MAX_BOUNDARY_LENGTH = 70; // As defined by RFC1341 + static constexpr int64 MAX_FILE_SIZE = 1500 << 20; // Telegram server file size limit + static constexpr const char TEMP_DIRECTORY_PREFIX[] = "tdlib-server-tmp"; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/NetStats.h b/protocols/Telegram/tdlib/td/tdnet/td/net/NetStats.h new file mode 100644 index 0000000000..e67f9fbc93 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/NetStats.h @@ -0,0 +1,145 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/SchedulerLocalStorage.h" + +#include "td/utils/common.h" +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/Time.h" + +#include <atomic> +#include <memory> + +namespace td { + +class NetStatsCallback { + public: + virtual void on_read(uint64 bytes) = 0; + virtual void on_write(uint64 bytes) = 0; + NetStatsCallback() = default; + NetStatsCallback(const NetStatsCallback &) = delete; + NetStatsCallback &operator=(const NetStatsCallback &) = delete; + virtual ~NetStatsCallback() = default; +}; + +struct NetStatsData { + uint64 read_size = 0; + uint64 write_size = 0; + + uint64 count = 0; + double duration = 0; +}; + +inline NetStatsData operator+(const NetStatsData &a, const NetStatsData &b) { + NetStatsData res; + res.read_size = a.read_size + b.read_size; + res.write_size = a.write_size + b.write_size; + res.count = a.count + b.count; + res.duration = a.duration + b.duration; + return res; +} +inline NetStatsData operator-(const NetStatsData &a, const NetStatsData &b) { + NetStatsData res; + CHECK(a.read_size >= b.read_size); + res.read_size = a.read_size - b.read_size; + + CHECK(a.write_size >= b.write_size); + res.write_size = a.write_size - b.write_size; + + CHECK(a.count >= b.count); + res.count = a.count - b.count; + + CHECK(a.duration >= b.duration); + res.duration = a.duration - b.duration; + + return res; +} + +inline StringBuilder &operator<<(StringBuilder &sb, const NetStatsData &data) { + return sb << tag("Rx size", format::as_size(data.read_size)) << tag("Tx size", format::as_size(data.write_size)) + << tag("count", data.count) << tag("duration", format::as_time(data.duration)); +} + +class NetStats { + public: + class Callback { + public: + virtual void on_stats_updated() = 0; + Callback() = default; + Callback(const Callback &) = delete; + Callback &operator=(const Callback &) = delete; + virtual ~Callback() = default; + }; + + std::shared_ptr<NetStatsCallback> get_callback() const { + return impl_; + } + + NetStatsData get_stats() const { + return impl_->get_stats(); + } + + // do it before get_callback + void set_callback(std::unique_ptr<Callback> callback) { + impl_->set_callback(std::move(callback)); + } + + private: + class Impl : public NetStatsCallback { + public: + NetStatsData get_stats() const { + NetStatsData res; + local_net_stats_.for_each([&](auto &stats) { + res.read_size += stats.read_size.load(std::memory_order_relaxed); + res.write_size += stats.write_size.load(std::memory_order_relaxed); + }); + return res; + } + void set_callback(std::unique_ptr<Callback> callback) { + callback_ = std::move(callback); + } + + private: + struct LocalNetStats { + double last_update = 0; + uint64 unsync_size = 0; + std::atomic<uint64> read_size{0}; + std::atomic<uint64> write_size{0}; + }; + SchedulerLocalStorage<LocalNetStats> local_net_stats_; + std::unique_ptr<Callback> callback_; + + void on_read(uint64 size) final { + auto &stats = local_net_stats_.get(); + stats.read_size.fetch_add(size, std::memory_order_relaxed); + + on_change(stats, size); + } + void on_write(uint64 size) final { + auto &stats = local_net_stats_.get(); + stats.write_size.fetch_add(size, std::memory_order_relaxed); + + on_change(stats, size); + } + + void on_change(LocalNetStats &stats, uint64 size) { + stats.unsync_size += size; + auto now = Time::now_cached(); + if (stats.unsync_size > 10000 || now - stats.last_update > 5 * 60) { + stats.unsync_size = 0; + stats.last_update = now; + callback_->on_stats_updated(); + } + } + }; + std::shared_ptr<Impl> impl_{std::make_shared<Impl>()}; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/Socks5.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/Socks5.cpp new file mode 100644 index 0000000000..02e1e067ea --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/Socks5.cpp @@ -0,0 +1,249 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/Socks5.h" + +#include "td/utils/format.h" +#include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/Fd.h" +#include "td/utils/Slice.h" + +namespace td { + +static int VERBOSITY_NAME(socks5) = VERBOSITY_NAME(DEBUG); + +Socks5::Socks5(SocketFd socket_fd, IPAddress ip_address, string username, string password, + std::unique_ptr<Callback> callback, ActorShared<> parent) + : fd_(std::move(socket_fd)) + , ip_address_(std::move(ip_address)) + , username_(std::move(username)) + , password_(std::move(password)) + , callback_(std::move(callback)) + , parent_(std::move(parent)) { +} + +void Socks5::on_error(Status status) { + CHECK(status.is_error()); + VLOG(socks5) << "Receive " << status; + if (callback_) { + callback_->set_result(std::move(status)); + callback_.reset(); + } + stop(); +} + +void Socks5::tear_down() { + VLOG(socks5) << "Finish to connect to proxy"; + unsubscribe(fd_.get_fd()); + fd_.get_fd().set_observer(nullptr); + if (callback_) { + callback_->set_result(std::move(fd_)); + callback_.reset(); + } +} + +void Socks5::hangup() { + on_error(Status::Error("Cancelled")); +} + +void Socks5::start_up() { + VLOG(socks5) << "Begin to connect to proxy"; + fd_.get_fd().set_observer(this); + subscribe(fd_.get_fd()); + set_timeout_in(10); + if (can_write(fd_)) { + loop(); + } +} + +void Socks5::send_greeting() { + VLOG(socks5) << "Send greeting to proxy"; + CHECK(state_ == State::SendGreeting); + state_ = State::WaitGreetingResponse; + + string greeting; + greeting += '\x05'; + bool use_username = !username_.empty(); + char authentication_count = use_username ? '\x02' : '\x01'; + greeting += authentication_count; + greeting += '\0'; + if (use_username) { + greeting += '\x02'; + } + + fd_.output_buffer().append(greeting); +} + +Status Socks5::wait_greeting_response() { + auto &buf = fd_.input_buffer(); + VLOG(socks5) << "Receive greeting response of size " << buf.size(); + if (buf.size() < 2) { + return Status::OK(); + } + auto buffer_slice = buf.read_as_buffer_slice(2); + auto slice = buffer_slice.as_slice(); + if (slice[0] != '\x05') { + return Status::Error(PSLICE() << "Unsupported socks protocol version " << int(slice[0])); + } + auto authentication_method = slice[1]; + if (authentication_method == '\0') { + state_ = State::SendIpAddress; + send_ip_address(); + return Status::OK(); + } + if (authentication_method == '\x02') { + return send_username_password(); + } + return Status::Error("Unsupported authentication mode"); +} + +Status Socks5::send_username_password() { + VLOG(socks5) << "Send username and password"; + if (username_.size() >= 128) { + return Status::Error("Username is too long"); + } + if (password_.size() >= 128) { + return Status::Error("Password is too long"); + } + + string request; + request += '\x01'; + request += narrow_cast<char>(username_.size()); + request += username_; + request += narrow_cast<char>(password_.size()); + request += password_; + fd_.output_buffer().append(request); + state_ = State::WaitPasswordResponse; + + return Status::OK(); +} + +Status Socks5::wait_password_response() { + auto &buf = fd_.input_buffer(); + VLOG(socks5) << "Receive password response of size " << buf.size(); + if (buf.size() < 2) { + return Status::OK(); + } + auto buffer_slice = buf.read_as_buffer_slice(2); + auto slice = buffer_slice.as_slice(); + if (slice[0] != '\x01') { + return Status::Error(PSLICE() << "Unsupported socks subnegotiation protocol version " << int(slice[0])); + } + if (slice[1] != '\x00') { + return Status::Error("Wrong username or password"); + } + + state_ = State::SendIpAddress; + send_ip_address(); + return Status::OK(); +} + +void Socks5::send_ip_address() { + VLOG(socks5) << "Send IP address"; + CHECK(state_ == State::SendIpAddress); + callback_->on_connected(); + string request; + request += '\x05'; + request += '\x01'; + request += '\x00'; + if (ip_address_.is_ipv4()) { + request += '\x01'; + auto ipv4 = ip_address_.get_ipv4(); + request += static_cast<char>(ipv4 & 255); + request += static_cast<char>((ipv4 >> 8) & 255); + request += static_cast<char>((ipv4 >> 16) & 255); + request += static_cast<char>((ipv4 >> 24) & 255); + } else { + request += '\x04'; + request += ip_address_.get_ipv6().str(); + } + auto port = ip_address_.get_port(); + request += static_cast<char>((port >> 8) & 255); + request += static_cast<char>(port & 255); + fd_.output_buffer().append(request); + state_ = State::WaitIpAddressResponse; +} + +Status Socks5::wait_ip_address_response() { + CHECK(state_ == State::WaitIpAddressResponse); + auto it = fd_.input_buffer().clone(); + VLOG(socks5) << "Receive IP address response of size " << it.size(); + if (it.size() < 4) { + return Status::OK(); + } + char c; + MutableSlice c_slice(&c, 1); + it.advance(1, c_slice); + if (c != '\x05') { + return Status::Error("Invalid response"); + } + it.advance(1, c_slice); + if (c != '\0') { + return Status::Error(PSLICE() << tag("code", c)); + } + it.advance(1, c_slice); + if (c != '\0') { + return Status::Error("byte must be zero"); + } + it.advance(1, c_slice); + if (c == '\x01') { + if (it.size() < 4) { + return Status::OK(); + } + it.advance(4); + } else if (c == '\x04') { + if (it.size() < 16) { + return Status::OK(); + } + it.advance(16); + } else { + return Status::Error("Invalid response"); + } + if (it.size() < 2) { + return Status::OK(); + } + it.advance(2); + stop(); + return Status::OK(); +} + +void Socks5::loop() { + auto status = [&] { + TRY_STATUS(fd_.flush_read()); + switch (state_) { + case State::SendGreeting: + send_greeting(); + break; + case State::WaitGreetingResponse: + TRY_STATUS(wait_greeting_response()); + break; + case State::WaitPasswordResponse: + TRY_STATUS(wait_password_response()); + break; + case State::WaitIpAddressResponse: + TRY_STATUS(wait_ip_address_response()); + break; + case State::SendIpAddress: + case State::Stop: + UNREACHABLE(); + } + TRY_STATUS(fd_.flush_write()); + return Status::OK(); + }(); + if (status.is_error()) { + on_error(std::move(status)); + } + if (can_close(fd_)) { + on_error(Status::Error("Connection closed")); + } +} + +void Socks5::timeout_expired() { + on_error(Status::Error("Timeout expired")); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/Socks5.h b/protocols/Telegram/tdlib/td/tdnet/td/net/Socks5.h new file mode 100644 index 0000000000..b67a33c282 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/Socks5.h @@ -0,0 +1,71 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" +#include "td/actor/PromiseFuture.h" + +#include "td/utils/BufferedFd.h" +#include "td/utils/common.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/Status.h" + +namespace td { + +class Socks5 : public Actor { + public: + class Callback { + public: + Callback() = default; + Callback(const Callback &) = delete; + Callback &operator=(const Callback &) = delete; + virtual ~Callback() = default; + + virtual void set_result(Result<SocketFd>) = 0; + virtual void on_connected() = 0; + }; + + Socks5(SocketFd socket_fd, IPAddress ip_address, string username, string password, std::unique_ptr<Callback> callback, + ActorShared<> parent); + + private: + BufferedFd<SocketFd> fd_; + IPAddress ip_address_; + string username_; + string password_; + std::unique_ptr<Callback> callback_; + ActorShared<> parent_; + + void on_error(Status status); + void tear_down() override; + void start_up() override; + void hangup() override; + + enum class State { + SendGreeting, + WaitGreetingResponse, + WaitPasswordResponse, + SendIpAddress, + WaitIpAddressResponse, + Stop + } state_ = State::SendGreeting; + + void send_greeting(); + Status wait_greeting_response(); + Status send_username_password(); + + Status wait_password_response(); + + void send_ip_address(); + Status wait_ip_address_response(); + + void loop() override; + void timeout_expired() override; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/SslFd.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/SslFd.cpp new file mode 100644 index 0000000000..f6f7557235 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/SslFd.cpp @@ -0,0 +1,280 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/SslFd.h" + +#include "td/utils/logging.h" +#include "td/utils/StackAllocator.h" +#include "td/utils/StringBuilder.h" +#include "td/utils/Time.h" + +#include <openssl/err.h> +#include <openssl/evp.h> +#include <openssl/ssl.h> +#include <openssl/x509v3.h> + +#include <map> +#include <mutex> + +namespace td { + +#if !TD_WINDOWS +static int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { + if (!preverify_ok) { + char buf[256]; + X509_NAME_oneline(X509_get_subject_name(X509_STORE_CTX_get_current_cert(ctx)), buf, 256); + + int err = X509_STORE_CTX_get_error(ctx); + auto warning = PSTRING() << "verify error:num=" << err << ":" << X509_verify_cert_error_string(err) + << ":depth=" << X509_STORE_CTX_get_error_depth(ctx) << ":" << buf; + double now = Time::now(); + + static std::mutex warning_mutex; + { + std::lock_guard<std::mutex> lock(warning_mutex); + static std::map<std::string, double> next_warning_time; + double &next = next_warning_time[warning]; + if (next <= now) { + next = now + 300; // one warning per 5 minutes + LOG(WARNING) << warning; + } + } + } + + return preverify_ok; +} +#endif + +namespace { + +Status create_openssl_error(int code, Slice message) { + const int buf_size = 1 << 12; + auto buf = StackAllocator::alloc(buf_size); + StringBuilder sb(buf.as_slice()); + + sb << message; + while (unsigned long error_code = ERR_get_error()) { + sb << "{" << error_code << ", " << ERR_error_string(error_code, nullptr) << "}"; + } + LOG_IF(ERROR, sb.is_error()) << "OPENSSL error buffer overflow"; + return Status::Error(code, sb.as_cslice()); +} + +void openssl_clear_errors(Slice from) { + if (ERR_peek_error() != 0) { + LOG(ERROR) << from << ": " << create_openssl_error(0, "Unprocessed OPENSSL_ERROR"); + } + errno = 0; +} + +void do_ssl_shutdown(SSL *ssl_handle) { + if (!SSL_is_init_finished(ssl_handle)) { + return; + } + openssl_clear_errors("Before SSL_shutdown"); + SSL_set_quiet_shutdown(ssl_handle, 1); + SSL_shutdown(ssl_handle); + openssl_clear_errors("After SSL_shutdown"); +} + +} // namespace + +SslFd::SslFd(SocketFd &&fd, SSL *ssl_handle_, SSL_CTX *ssl_ctx_) + : fd_(std::move(fd)), ssl_handle_(ssl_handle_), ssl_ctx_(ssl_ctx_) { +} + +Result<SslFd> SslFd::init(SocketFd fd, CSlice host, CSlice cert_file, VerifyPeer verify_peer) { +#if TD_WINDOWS + return Status::Error("TODO"); +#else + static bool init_openssl = [] { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + return OPENSSL_init_ssl(0, nullptr) != 0; +#else + OpenSSL_add_all_algorithms(); + SSL_load_error_strings(); + return OpenSSL_add_ssl_algorithms() != 0; +#endif + }(); + CHECK(init_openssl); + + openssl_clear_errors("Before SslFd::init"); + CHECK(!fd.empty()); + + auto ssl_method = +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + TLS_client_method(); +#else + SSLv23_client_method(); +#endif + if (ssl_method == nullptr) { + return create_openssl_error(-6, "Failed to create an SSL client method"); + } + + auto ssl_ctx = SSL_CTX_new(ssl_method); + if (ssl_ctx == nullptr) { + return create_openssl_error(-7, "Failed to create an SSL context"); + } + auto ssl_ctx_guard = ScopeExit() + [&]() { SSL_CTX_free(ssl_ctx); }; + long options = 0; +#ifdef SSL_OP_NO_SSLv2 + options |= SSL_OP_NO_SSLv2; +#endif +#ifdef SSL_OP_NO_SSLv3 + options |= SSL_OP_NO_SSLv3; +#endif + SSL_CTX_set_options(ssl_ctx, options); + SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); + + if (cert_file.empty()) { + SSL_CTX_set_default_verify_paths(ssl_ctx); + } else { + if (SSL_CTX_load_verify_locations(ssl_ctx, cert_file.c_str(), nullptr) == 0) { + return create_openssl_error(-8, "Failed to set custom cert file"); + } + } + if (VERIFY_PEER && verify_peer == VerifyPeer::On) { + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, verify_callback); + + if (VERIFY_DEPTH != -1) { + SSL_CTX_set_verify_depth(ssl_ctx, VERIFY_DEPTH); + } + } else { + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_NONE, nullptr); + } + + // TODO(now): cipher list + string cipher_list; + if (SSL_CTX_set_cipher_list(ssl_ctx, cipher_list.empty() ? "DEFAULT" : cipher_list.c_str()) == 0) { + return create_openssl_error(-9, PSLICE() << "Failed to set cipher list \"" << cipher_list << '"'); + } + + auto ssl_handle = SSL_new(ssl_ctx); + if (ssl_handle == nullptr) { + return create_openssl_error(-13, "Failed to create an SSL handle"); + } + auto ssl_handle_guard = ScopeExit() + [&]() { + do_ssl_shutdown(ssl_handle); + SSL_free(ssl_handle); + }; + +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + X509_VERIFY_PARAM *param = SSL_get0_param(ssl_handle); + /* Enable automatic hostname checks */ + // TODO: X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS + X509_VERIFY_PARAM_set_hostflags(param, 0); + X509_VERIFY_PARAM_set1_host(param, host.c_str(), 0); +#else +#warning DANGEROUS! HTTPS HOST WILL NOT BE CHECKED. INSTALL OPENSSL >= 1.0.2 OR IMPLEMENT HTTPS HOST CHECK MANUALLY +#endif + + if (!SSL_set_fd(ssl_handle, fd.get_fd().get_native_fd())) { + return create_openssl_error(-14, "Failed to set fd"); + } + +#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) + auto host_str = host.str(); + SSL_set_tlsext_host_name(ssl_handle, MutableCSlice(host_str).begin()); +#endif + SSL_set_connect_state(ssl_handle); + + ssl_ctx_guard.dismiss(); + ssl_handle_guard.dismiss(); + return SslFd(std::move(fd), ssl_handle, ssl_ctx); +#endif +} + +Result<size_t> SslFd::process_ssl_error(int ret, int *mask) { +#if TD_WINDOWS + return Status::Error("TODO"); +#else + auto openssl_errno = errno; + int error = SSL_get_error(ssl_handle_, ret); + LOG(INFO) << "SSL ERROR: " << ret << " " << error; + switch (error) { + case SSL_ERROR_NONE: + LOG(ERROR) << "SSL_get_error returned no error"; + return 0; + case SSL_ERROR_ZERO_RETURN: + LOG(DEBUG) << "SSL_ERROR_ZERO_RETURN"; + fd_.get_fd().update_flags(Fd::Close); + write_mask_ |= Fd::Error; + *mask |= Fd::Error; + return 0; + case SSL_ERROR_WANT_READ: + LOG(DEBUG) << "SSL_ERROR_WANT_READ"; + fd_.get_fd().clear_flags(Fd::Read); + *mask |= Fd::Read; + return 0; + case SSL_ERROR_WANT_WRITE: + LOG(DEBUG) << "SSL_ERROR_WANT_WRITE"; + fd_.get_fd().clear_flags(Fd::Write); + *mask |= Fd::Write; + return 0; + case SSL_ERROR_WANT_CONNECT: + case SSL_ERROR_WANT_ACCEPT: + case SSL_ERROR_WANT_X509_LOOKUP: + LOG(DEBUG) << "SSL_ERROR: CONNECT ACCEPT LOOKUP"; + fd_.get_fd().clear_flags(Fd::Write); + *mask |= Fd::Write; + return 0; + case SSL_ERROR_SYSCALL: + LOG(DEBUG) << "SSL_ERROR_SYSCALL"; + if (ERR_peek_error() == 0) { + if (openssl_errno != 0) { + CHECK(openssl_errno != EAGAIN); + return Status::PosixError(openssl_errno, "SSL_ERROR_SYSCALL"); + } else { + // Socket was closed from the other side, probably. Not an error + fd_.get_fd().update_flags(Fd::Close); + write_mask_ |= Fd::Error; + *mask |= Fd::Error; + return 0; + } + } + /* fall through */ + default: + LOG(DEBUG) << "SSL_ERROR Default"; + fd_.get_fd().update_flags(Fd::Close); + write_mask_ |= Fd::Error; + read_mask_ |= Fd::Error; + return create_openssl_error(1, "SSL error "); + } +#endif +} + +Result<size_t> SslFd::write(Slice slice) { + openssl_clear_errors("Before SslFd::write"); + auto size = SSL_write(ssl_handle_, slice.data(), static_cast<int>(slice.size())); + if (size <= 0) { + return process_ssl_error(size, &write_mask_); + } + return size; +} +Result<size_t> SslFd::read(MutableSlice slice) { + openssl_clear_errors("Before SslFd::read"); + auto size = SSL_read(ssl_handle_, slice.data(), static_cast<int>(slice.size())); + if (size <= 0) { + return process_ssl_error(size, &read_mask_); + } + return size; +} + +void SslFd::close() { + if (fd_.empty()) { + CHECK(!ssl_handle_ && !ssl_ctx_); + return; + } + CHECK(ssl_handle_ && ssl_ctx_); + do_ssl_shutdown(ssl_handle_); + SSL_free(ssl_handle_); + ssl_handle_ = nullptr; + SSL_CTX_free(ssl_ctx_); + ssl_ctx_ = nullptr; + fd_.close(); +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/SslFd.h b/protocols/Telegram/tdlib/td/tdnet/td/net/SslFd.h new file mode 100644 index 0000000000..c197b9c318 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/SslFd.h @@ -0,0 +1,109 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/utils/port/Fd.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +#include <openssl/ssl.h> // TODO can we remove it from header and make target_link_libraries dependence PRIVATE? + +namespace td { + +class SslFd { + public: + enum class VerifyPeer { On, Off }; + static Result<SslFd> init(SocketFd fd, CSlice host, CSlice cert_file = CSlice(), + VerifyPeer verify_peer = VerifyPeer::On) TD_WARN_UNUSED_RESULT; + + SslFd(const SslFd &other) = delete; + SslFd &operator=(const SslFd &other) = delete; + SslFd(SslFd &&other) + : fd_(std::move(other.fd_)) + , write_mask_(other.write_mask_) + , read_mask_(other.read_mask_) + , ssl_handle_(other.ssl_handle_) + , ssl_ctx_(other.ssl_ctx_) { + other.ssl_handle_ = nullptr; + other.ssl_ctx_ = nullptr; + } + SslFd &operator=(SslFd &&other) { + close(); + + fd_ = std::move(other.fd_); + write_mask_ = other.write_mask_; + read_mask_ = other.read_mask_; + ssl_handle_ = other.ssl_handle_; + ssl_ctx_ = other.ssl_ctx_; + + other.ssl_handle_ = nullptr; + other.ssl_ctx_ = nullptr; + return *this; + } + + const Fd &get_fd() const { + return fd_.get_fd(); + } + + Fd &get_fd() { + return fd_.get_fd(); + } + + Status get_pending_error() TD_WARN_UNUSED_RESULT { + return fd_.get_pending_error(); + } + + Result<size_t> write(Slice slice) TD_WARN_UNUSED_RESULT; + Result<size_t> read(MutableSlice slice) TD_WARN_UNUSED_RESULT; + + void close(); + + int32 get_flags() const { + int32 res = 0; + int32 fd_flags = fd_.get_flags(); + fd_flags &= ~Fd::Error; + if (fd_flags & Fd::Close) { + res |= Fd::Close; + } + write_mask_ &= ~fd_flags; + read_mask_ &= ~fd_flags; + if (write_mask_ == 0) { + res |= Fd::Write; + } + if (read_mask_ == 0) { + res |= Fd::Read; + } + return res; + } + + bool empty() const { + return fd_.empty(); + } + + ~SslFd() { + close(); + } + + private: + static constexpr bool VERIFY_PEER = true; + static constexpr int VERIFY_DEPTH = 10; + + SocketFd fd_; + mutable int write_mask_ = 0; + mutable int read_mask_ = 0; + + // TODO unique_ptr + SSL *ssl_handle_ = nullptr; + SSL_CTX *ssl_ctx_ = nullptr; + + SslFd(SocketFd &&fd, SSL *ssl_handle_, SSL_CTX *ssl_ctx_); + + Result<size_t> process_ssl_error(int ret, int *mask) TD_WARN_UNUSED_RESULT; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/TcpListener.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/TcpListener.cpp new file mode 100644 index 0000000000..54531f9b60 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/TcpListener.cpp @@ -0,0 +1,62 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/TcpListener.h" + +#include "td/utils/logging.h" +#include "td/utils/port/Fd.h" + +namespace td { +// TcpListener implementation +TcpListener::TcpListener(int port, ActorShared<Callback> callback) : port_(port), callback_(std::move(callback)) { +} + +void TcpListener::hangup() { + stop(); +} + +void TcpListener::start_up() { + auto r_socket = ServerSocketFd::open(port_); + if (r_socket.is_error()) { + LOG(ERROR) << "Can't open server socket: " << r_socket.error(); + set_timeout_in(5); + return; + } + server_fd_ = r_socket.move_as_ok(); + server_fd_.get_fd().set_observer(this); + subscribe(server_fd_.get_fd()); +} + +void TcpListener::tear_down() { + LOG(ERROR) << "TcpListener closed"; + if (!server_fd_.empty()) { + unsubscribe_before_close(server_fd_.get_fd()); + server_fd_.close(); + } +} + +void TcpListener::loop() { + if (server_fd_.empty()) { + start_up(); + } + while (can_read(server_fd_)) { + auto r_socket_fd = server_fd_.accept(); + if (r_socket_fd.is_error()) { + if (r_socket_fd.error().code() != -1) { + LOG(ERROR) << r_socket_fd.error(); + } + continue; + } + send_closure(callback_, &Callback::accept, r_socket_fd.move_as_ok()); + } + + if (can_close(server_fd_)) { + LOG(ERROR) << "HELLO!"; + stop(); + } +} + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/TcpListener.h b/protocols/Telegram/tdlib/td/tdnet/td/net/TcpListener.h new file mode 100644 index 0000000000..f2e61a2387 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/TcpListener.h @@ -0,0 +1,35 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/actor/actor.h" + +#include "td/utils/port/ServerSocketFd.h" +#include "td/utils/port/SocketFd.h" + +namespace td { + +class TcpListener final : public Actor { + public: + class Callback : public Actor { + public: + virtual void accept(SocketFd fd) = 0; + }; + + TcpListener(int port, ActorShared<Callback> callback); + void hangup() override; + + private: + int port_; + ServerSocketFd server_fd_; + ActorShared<Callback> callback_; + void start_up() override; + void tear_down() override; + void loop() override; +}; + +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/Wget.cpp b/protocols/Telegram/tdlib/td/tdnet/td/net/Wget.cpp new file mode 100644 index 0000000000..b30128be32 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/Wget.cpp @@ -0,0 +1,126 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#include "td/net/Wget.h" + +#include "td/net/HttpHeaderCreator.h" +#include "td/net/HttpOutboundConnection.h" +#include "td/net/SslFd.h" + +#include "td/utils/buffer.h" +#include "td/utils/HttpUrl.h" +#include "td/utils/logging.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/port/SocketFd.h" +#include "td/utils/Slice.h" + +#include <limits> + +namespace td { +Wget::Wget(Promise<HttpQueryPtr> promise, string url, std::vector<std::pair<string, string>> headers, int32 timeout_in, + int32 ttl, SslFd::VerifyPeer verify_peer) + : promise_(std::move(promise)) + , input_url_(std::move(url)) + , headers_(std::move(headers)) + , timeout_in_(timeout_in) + , ttl_(ttl) + , verify_peer_(verify_peer) { +} + +Status Wget::try_init() { + string input_url = input_url_; + TRY_RESULT(url, parse_url(MutableSlice(input_url))); + + IPAddress addr; + TRY_STATUS(addr.init_host_port(url.host_, url.port_)); + + TRY_RESULT(fd, SocketFd::open(addr)); + if (url.protocol_ == HttpUrl::Protocol::HTTP) { + connection_ = + create_actor<HttpOutboundConnection>("Connect", std::move(fd), std::numeric_limits<std::size_t>::max(), 0, 0, + ActorOwn<HttpOutboundConnection::Callback>(actor_id(this))); + } else { + TRY_RESULT(ssl_fd, SslFd::init(std::move(fd), url.host_, CSlice() /* certificate */, verify_peer_)); + connection_ = + create_actor<HttpOutboundConnection>("Connect", std::move(ssl_fd), std::numeric_limits<std::size_t>::max(), 0, + 0, ActorOwn<HttpOutboundConnection::Callback>(actor_id(this))); + } + + HttpHeaderCreator hc; + hc.init_get(url.query_); + bool was_host = false; + for (auto &header : headers_) { + if (header.first == "Host") { // TODO: lowercase + was_host = true; + } + hc.add_header(header.first, header.second); + } + if (!was_host) { + hc.add_header("Host", url.host_); + } + hc.add_header("Accept-Encoding", "gzip, deflate"); + + send_closure(connection_, &HttpOutboundConnection::write_next, BufferSlice(hc.finish().ok())); + send_closure(connection_, &HttpOutboundConnection::write_ok); + return Status::OK(); +} + +void Wget::loop() { + if (connection_.empty()) { + auto status = try_init(); + if (status.is_error()) { + return on_error(std::move(status)); + } + } +} + +void Wget::handle(HttpQueryPtr result) { + on_ok(std::move(result)); +} + +void Wget::on_connection_error(Status error) { + on_error(std::move(error)); +} + +void Wget::on_ok(HttpQueryPtr http_query_ptr) { + CHECK(promise_); + if (http_query_ptr->code_ == 302 && ttl_ > 0) { + LOG(DEBUG) << *http_query_ptr; + input_url_ = http_query_ptr->header("location").str(); + LOG(DEBUG) << input_url_; + ttl_--; + connection_.reset(); + yield(); + } else if (http_query_ptr->code_ >= 200 && http_query_ptr->code_ < 300) { + promise_.set_value(std::move(http_query_ptr)); + stop(); + } else { + on_error(Status::Error(PSLICE() << "http error: " << http_query_ptr->code_)); + } +} + +void Wget::on_error(Status error) { + CHECK(error.is_error()); + CHECK(promise_); + promise_.set_error(std::move(error)); + stop(); +} + +void Wget::start_up() { + set_timeout_in(timeout_in_); + loop(); +} + +void Wget::timeout_expired() { + on_error(Status::Error("Timeout expired")); +} + +void Wget::tear_down() { + if (promise_) { + on_error(Status::Error("Cancelled")); + } +} +} // namespace td diff --git a/protocols/Telegram/tdlib/td/tdnet/td/net/Wget.h b/protocols/Telegram/tdlib/td/tdnet/td/net/Wget.h new file mode 100644 index 0000000000..cecb113c94 --- /dev/null +++ b/protocols/Telegram/tdlib/td/tdnet/td/net/Wget.h @@ -0,0 +1,48 @@ +// +// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018 +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +#pragma once + +#include "td/net/HttpOutboundConnection.h" +#include "td/net/HttpQuery.h" +#include "td/net/SslFd.h" + +#include "td/actor/PromiseFuture.h" + +#include "td/utils/common.h" +#include "td/utils/Status.h" + +#include <utility> + +namespace td { + +class Wget : public HttpOutboundConnection::Callback { + public: + explicit Wget(Promise<HttpQueryPtr> promise, string url, std::vector<std::pair<string, string>> headers = {}, + int32 timeout_in = 10, int32 ttl = 3, SslFd::VerifyPeer verify_peer = SslFd::VerifyPeer::On); + + private: + Status try_init(); + void loop() override; + void handle(HttpQueryPtr result) override; + void on_connection_error(Status error) override; + void on_ok(HttpQueryPtr http_query_ptr); + void on_error(Status error); + + void tear_down() override; + void start_up() override; + void timeout_expired() override; + + Promise<HttpQueryPtr> promise_; + ActorOwn<HttpOutboundConnection> connection_; + string input_url_; + std::vector<std::pair<string, string>> headers_; + int32 timeout_in_; + int32 ttl_; + SslFd::VerifyPeer verify_peer_; +}; + +} // namespace td |