ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2026-04-29 15:37:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Integrate new webserver (#1722)

Browse Source 
For #919.
The new web server is not using dosguard yet. It will be fixed by a
separate PR.
This commit is contained in:
Sergey Kuznetsov
2024-11-21 14:48:32 +00:00
committed by GitHub
parent fc3ba07f2e
commit c77154a5e6
90 changed files with 4029 additions and 683 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
src/web/ng/impl/ConnectionHandler.cpp
View File

@@ -21,21 +21,30 @@
#include "util/Assert.hpp"
#include "util/CoroutineGroup.hpp"
#include "util/Taggable.hpp"
#include "util/log/Logger.hpp"
#include "web/SubscriptionContextInterface.hpp"
#include "web/ng/Connection.hpp"
#include "web/ng/Error.hpp"
#include "web/ng/MessageHandler.hpp"
#include "web/ng/ProcessingPolicy.hpp"
#include "web/ng/Request.hpp"
#include "web/ng/Response.hpp"
#include "web/ng/SubscriptionContext.hpp"
#include <boost/asio/bind_cancellation_slot.hpp>
#include <boost/asio/cancellation_signal.hpp>
#include <boost/asio/error.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/ssl/error.hpp>
#include <boost/asio/steady_timer.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/http/error.hpp>
#include <boost/beast/http/status.hpp>
#include <boost/beast/websocket/error.hpp>
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
@@ -47,7 +56,8 @@ namespace {
Response
handleHttpRequest(
ConnectionContext const& connectionContext,
ConnectionMetadata& connectionMetadata,
SubscriptionContextPtr& subscriptionContext,
ConnectionHandler::TargetToHandlerMap const& handlers,
Request const& request,
boost::asio::yield_context yield
@@ -58,12 +68,13 @@ handleHttpRequest(
if (it == handlers.end()) {
return Response{boost::beast::http::status::bad_request, "Bad target", request};
}
return it->second(request, connectionContext, yield);
return it->second(request, connectionMetadata, subscriptionContext, yield);
}
Response
handleWsRequest(
ConnectionContext connectionContext,
ConnectionMetadata& connectionMetadata,
SubscriptionContextPtr& subscriptionContext,
std::optional<MessageHandler> const& handler,
Request const& request,
boost::asio::yield_context yield
@@ -72,7 +83,7 @@ handleWsRequest(
if (not handler.has_value()) {
return Response{boost::beast::http::status::bad_request, "WebSocket is not supported by this server", request};
}
return handler->operator()(request, connectionContext, yield);
return handler->operator()(request, connectionMetadata, subscriptionContext, yield);
}
} // namespace
@@ -95,8 +106,16 @@ ConnectionHandler::StringHash::operator()(std::string const& str) const
return hash_type{}(str);
}
ConnectionHandler::ConnectionHandler(ProcessingPolicy processingPolicy, std::optional<size_t> maxParallelRequests)
: processingPolicy_{processingPolicy}, maxParallelRequests_{maxParallelRequests}
ConnectionHandler::ConnectionHandler(
ProcessingPolicy processingPolicy,
std::optional<size_t> maxParallelRequests,
util::TagDecoratorFactory& tagFactory,
std::optional<size_t> maxSubscriptionSendQueueSize
)
: processingPolicy_{processingPolicy}
, maxParallelRequests_{maxParallelRequests}
, tagFactory_{tagFactory}
, maxSubscriptionSendQueueSize_{maxSubscriptionSendQueueSize}
{
}
@@ -126,14 +145,32 @@ ConnectionHandler::processConnection(ConnectionPtr connectionPtr, boost::asio::y
bool shouldCloseGracefully = false;
std::shared_ptr<SubscriptionContext> subscriptionContext;
if (connectionRef.wasUpgraded()) {
auto* ptr = dynamic_cast<impl::WsConnectionBase*>(connectionPtr.get());
ASSERT(ptr != nullptr, "Casted not websocket connection");
subscriptionContext = std::make_shared<SubscriptionContext>(
tagFactory_,
*ptr,
maxSubscriptionSendQueueSize_,
yield,
[this](Error const& e, Connection const& c) { return handleError(e, c); }
);
}
SubscriptionContextPtr subscriptionContextInterfacePtr = subscriptionContext;
switch (processingPolicy_) {
case ProcessingPolicy::Sequential:
shouldCloseGracefully = sequentRequestResponseLoop(connectionRef, yield);
shouldCloseGracefully = sequentRequestResponseLoop(connectionRef, subscriptionContextInterfacePtr, yield);
break;
case ProcessingPolicy::Parallel:
shouldCloseGracefully = parallelRequestResponseLoop(connectionRef, yield);
shouldCloseGracefully = parallelRequestResponseLoop(connectionRef, subscriptionContextInterfacePtr, yield);
break;
}
if (subscriptionContext != nullptr)
subscriptionContext->disconnect(yield);
if (shouldCloseGracefully)
connectionRef.close(yield);
@@ -179,7 +216,11 @@ ConnectionHandler::handleError(Error const& error, Connection const& connection)
}
bool
ConnectionHandler::sequentRequestResponseLoop(Connection& connection, boost::asio::yield_context yield)
ConnectionHandler::sequentRequestResponseLoop(
Connection& connection,
SubscriptionContextPtr& subscriptionContext,
boost::asio::yield_context yield
)
{
// The loop here is infinite because:
// - For websocket connection is persistent so Clio will try to read and respond infinite unless client
@@ -196,14 +237,19 @@ ConnectionHandler::sequentRequestResponseLoop(Connection& connection, boost::asi
LOG(log_.info()) << connection.tag() << "Received request from ip = " << connection.ip();
auto maybeReturnValue = processRequest(connection, std::move(expectedRequest).value(), yield);
auto maybeReturnValue =
processRequest(connection, subscriptionContext, std::move(expectedRequest).value(), yield);
if (maybeReturnValue.has_value())
return maybeReturnValue.value();
}
}
bool
ConnectionHandler::parallelRequestResponseLoop(Connection& connection, boost::asio::yield_context yield)
ConnectionHandler::parallelRequestResponseLoop(
Connection& connection,
SubscriptionContextPtr& subscriptionContext,
boost::asio::yield_context yield
)
{
// atomic_bool is not needed here because everything happening on coroutine's strand
bool stop = false;
@@ -218,13 +264,18 @@ ConnectionHandler::parallelRequestResponseLoop(Connection& connection, boost::as
closeConnectionGracefully &= closeGracefully;
break;
}
if (tasksGroup.canSpawn()) {
if (not tasksGroup.isFull()) {
bool const spawnSuccess = tasksGroup.spawn(
yield, // spawn on the same strand
[this, &stop, &closeConnectionGracefully, &connection, request = std::move(expectedRequest).value()](
boost::asio::yield_context innerYield
) mutable {
auto maybeCloseConnectionGracefully = processRequest(connection, request, innerYield);
[this,
&stop,
&closeConnectionGracefully,
&connection,
&subscriptionContext,
request = std::move(expectedRequest).value()](boost::asio::yield_context innerYield) mutable {
auto maybeCloseConnectionGracefully =
processRequest(connection, subscriptionContext, request, innerYield);
if (maybeCloseConnectionGracefully.has_value()) {
stop = true;
closeConnectionGracefully &= maybeCloseConnectionGracefully.value();
@@ -248,9 +299,14 @@ ConnectionHandler::parallelRequestResponseLoop(Connection& connection, boost::as
}
std::optional<bool>
ConnectionHandler::processRequest(Connection& connection, Request const& request, boost::asio::yield_context yield)
ConnectionHandler::processRequest(
Connection& connection,
SubscriptionContextPtr& subscriptionContext,
Request const& request,
boost::asio::yield_context yield
)
{
auto response = handleRequest(connection.context(), request, yield);
auto response = handleRequest(connection, subscriptionContext, request, yield);
auto const maybeError = connection.send(std::move(response), yield);
if (maybeError.has_value()) {
@@ -261,18 +317,19 @@ ConnectionHandler::processRequest(Connection& connection, Request const& request
Response
ConnectionHandler::handleRequest(
ConnectionContext const& connectionContext,
ConnectionMetadata& connectionMetadata,
SubscriptionContextPtr& subscriptionContext,
Request const& request,
boost::asio::yield_context yield
)
{
switch (request.method()) {
case Request::Method::Get:
return handleHttpRequest(connectionContext, getHandlers_, request, yield);
return handleHttpRequest(connectionMetadata, subscriptionContext, getHandlers_, request, yield);
case Request::Method::Post:
return handleHttpRequest(connectionContext, postHandlers_, request, yield);
return handleHttpRequest(connectionMetadata, subscriptionContext, postHandlers_, request, yield);
case Request::Method::Websocket:
return handleWsRequest(connectionContext, wsHandler_, request, yield);
return handleWsRequest(connectionMetadata, subscriptionContext, wsHandler_, request, yield);
default:
return Response{boost::beast::http::status::bad_request, "Unsupported http method", request};
}

49
src/web/ng/impl/ConnectionHandler.hpp
View File

@@ -19,10 +19,13 @@
#pragma once
#include "util/Taggable.hpp"
#include "util/log/Logger.hpp"
#include "web/SubscriptionContextInterface.hpp"
#include "web/ng/Connection.hpp"
#include "web/ng/Error.hpp"
#include "web/ng/MessageHandler.hpp"
#include "web/ng/ProcessingPolicy.hpp"
#include "web/ng/Request.hpp"
#include "web/ng/Response.hpp"
@@ -41,8 +44,6 @@ namespace web::ng::impl {
class ConnectionHandler {
public:
enum class ProcessingPolicy { Sequential, Parallel };
struct StringHash {
using hash_type = std::hash<std::string_view>;
using is_transparent = void;
@@ -64,6 +65,9 @@ private:
ProcessingPolicy processingPolicy_;
std::optional<size_t> maxParallelRequests_;
std::reference_wrapper<util::TagDecoratorFactory> tagFactory_;
std::optional<size_t> maxSubscriptionSendQueueSize_;
TargetToHandlerMap getHandlers_;
TargetToHandlerMap postHandlers_;
std::optional<MessageHandler> wsHandler_;
@@ -71,7 +75,12 @@ private:
boost::signals2::signal<void()> onStop_;
public:
ConnectionHandler(ProcessingPolicy processingPolicy, std::optional<size_t> maxParallelRequests);
ConnectionHandler(
ProcessingPolicy processingPolicy,
std::optional<size_t> maxParallelRequests,
util::TagDecoratorFactory& tagFactory,
std::optional<size_t> maxSubscriptionSendQueueSize
);
void
onGet(std::string const& target, MessageHandler handler);
@@ -107,24 +116,34 @@ private:
* @return True if the connection should be gracefully closed, false otherwise.
*/
bool
sequentRequestResponseLoop(Connection& connection, boost::asio::yield_context yield);
sequentRequestResponseLoop(
Connection& connection,
SubscriptionContextPtr& subscriptionContext,
boost::asio::yield_context yield
);
bool
parallelRequestResponseLoop(Connection& connection, boost::asio::yield_context yield);
parallelRequestResponseLoop(
Connection& connection,
SubscriptionContextPtr& subscriptionContext,
boost::asio::yield_context yield
);
std::optional<bool>
processRequest(Connection& connection, Request const& request, boost::asio::yield_context yield);
processRequest(
Connection& connection,
SubscriptionContextPtr& subscriptionContext,
Request const& request,
boost::asio::yield_context yield
);
/**
* @brief Handle a request.
*
* @param connectionContext The connection context.
* @param request The request to handle.
* @param yield The yield context.
* @return The response to send.
*/
Response
handleRequest(ConnectionContext const& connectionContext, Request const& request, boost::asio::yield_context yield);
handleRequest(
ConnectionMetadata& connectionMetadata,
SubscriptionContextPtr& subscriptionContext,
Request const& request,
boost::asio::yield_context yield
);
};
} // namespace web::ng::impl

165
src/web/ng/impl/ErrorHandling.cpp Normal file
View File

@@ -0,0 +1,165 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "web/ng/impl/ErrorHandling.hpp"
#include "rpc/Errors.hpp"
#include "rpc/JS.hpp"
#include "util/Assert.hpp"
#include "web/ng/Request.hpp"
#include "web/ng/Response.hpp"
#include <boost/beast/http/status.hpp>
#include <boost/json/object.hpp>
#include <fmt/core.h>
#include <xrpl/protocol/jss.h>
#include <optional>
#include <string>
#include <utility>
#include <variant>
namespace http = boost::beast::http;
namespace web::ng::impl {
namespace {
boost::json::object
composeErrorImpl(auto const& error, Request const& rawRequest, std::optional<boost::json::object> const& request)
{
auto e = rpc::makeError(error);
if (request) {
auto const appendFieldIfExist = [&](auto const& field) {
if (request->contains(field) and not request->at(field).is_null())
e[field] = request->at(field);
};
appendFieldIfExist(JS(id));
if (not rawRequest.isHttp())
appendFieldIfExist(JS(api_version));
e[JS(request)] = request.value();
}
if (not rawRequest.isHttp()) {
return e;
}
return {{JS(result), e}};
}
} // namespace
ErrorHelper::ErrorHelper(Request const& rawRequest, std::optional<boost::json::object> request)
: rawRequest_{rawRequest}, request_{std::move(request)}
{
}
Response
ErrorHelper::makeError(rpc::Status const& err) const
{
if (not rawRequest_.get().isHttp()) {
return Response{http::status::bad_request, composeError(err), rawRequest_};
}
// Note: a collection of crutches to match rippled output follows
if (auto const clioCode = std::get_if<rpc::ClioError>(&err.code)) {
switch (*clioCode) {
case rpc::ClioError::rpcINVALID_API_VERSION:
return Response{
http::status::bad_request, std::string{rpc::getErrorInfo(*clioCode).error}, rawRequest_
};
case rpc::ClioError::rpcCOMMAND_IS_MISSING:
return Response{http::status::bad_request, "Null method", rawRequest_};
case rpc::ClioError::rpcCOMMAND_IS_EMPTY:
return Response{http::status::bad_request, "method is empty", rawRequest_};
case rpc::ClioError::rpcCOMMAND_NOT_STRING:
return Response{http::status::bad_request, "method is not string", rawRequest_};
case rpc::ClioError::rpcPARAMS_UNPARSEABLE:
return Response{http::status::bad_request, "params unparseable", rawRequest_};
// others are not applicable but we want a compilation error next time we add one
case rpc::ClioError::rpcUNKNOWN_OPTION:
case rpc::ClioError::rpcMALFORMED_CURRENCY:
case rpc::ClioError::rpcMALFORMED_REQUEST:
case rpc::ClioError::rpcMALFORMED_OWNER:
case rpc::ClioError::rpcMALFORMED_ADDRESS:
case rpc::ClioError::rpcINVALID_HOT_WALLET:
case rpc::ClioError::rpcFIELD_NOT_FOUND_TRANSACTION:
case rpc::ClioError::rpcMALFORMED_ORACLE_DOCUMENT_ID:
case rpc::ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS:
case rpc::ClioError::etlCONNECTION_ERROR:
case rpc::ClioError::etlREQUEST_ERROR:
case rpc::ClioError::etlREQUEST_TIMEOUT:
case rpc::ClioError::etlINVALID_RESPONSE:
ASSERT(false, "Unknown rpc error code {}", static_cast<int>(*clioCode)); // this should never happen
break;
}
}
return Response{http::status::bad_request, composeError(err), rawRequest_};
}
Response
ErrorHelper::makeInternalError() const
{
return Response{http::status::internal_server_error, composeError(rpc::RippledError::rpcINTERNAL), rawRequest_};
}
Response
ErrorHelper::makeNotReadyError() const
{
return Response{http::status::ok, composeError(rpc::RippledError::rpcNOT_READY), rawRequest_};
}
Response
ErrorHelper::makeTooBusyError() const
{
if (not rawRequest_.get().isHttp()) {
return Response{http::status::too_many_requests, rpc::makeError(rpc::RippledError::rpcTOO_BUSY), rawRequest_};
}
return Response{http::status::service_unavailable, rpc::makeError(rpc::RippledError::rpcTOO_BUSY), rawRequest_};
}
Response
ErrorHelper::makeJsonParsingError() const
{
if (not rawRequest_.get().isHttp()) {
return Response{http::status::bad_request, rpc::makeError(rpc::RippledError::rpcBAD_SYNTAX), rawRequest_};
}
return Response{http::status::bad_request, fmt::format("Unable to parse JSON from the request"), rawRequest_};
}
boost::json::object
ErrorHelper::composeError(rpc::Status const& error) const
{
return composeErrorImpl(error, rawRequest_, request_);
}
boost::json::object
ErrorHelper::composeError(rpc::RippledError error) const
{
return composeErrorImpl(error, rawRequest_, request_);
}
} // namespace web::ng::impl

114
src/web/ng/impl/ErrorHandling.hpp Normal file
View File

@@ -0,0 +1,114 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "rpc/Errors.hpp"
#include "web/ng/Request.hpp"
#include "web/ng/Response.hpp"
#include <boost/beast/http/status.hpp>
#include <boost/json/object.hpp>
#include <boost/json/serialize.hpp>
#include <fmt/core.h>
#include <xrpl/protocol/ErrorCodes.h>
#include <xrpl/protocol/jss.h>
#include <functional>
#include <optional>
namespace web::ng::impl {
/**
* @brief A helper that attempts to match rippled reporting mode HTTP errors as close as possible.
*/
class ErrorHelper {
std::reference_wrapper<Request const> rawRequest_;
std::optional<boost::json::object> request_;
public:
/**
* @brief Construct a new Error Helper object
*
* @param rawRequest The request that caused the error.
* @param request The parsed request that caused the error.
*/
ErrorHelper(Request const& rawRequest, std::optional<boost::json::object> request = std::nullopt);
/**
* @brief Make an error response from a status.
*
* @param err The status to make an error response from.
* @return
*/
[[nodiscard]] Response
makeError(rpc::Status const& err) const;
/**
* @brief Make an internal error response.
*
* @return A response with an internal error.
*/
[[nodiscard]] Response
makeInternalError() const;
/**
* @brief Make a response for when the server is not ready.
*
* @return A response with a not ready error.
*/
[[nodiscard]] Response
makeNotReadyError() const;
/**
* @brief Make a response for when the server is too busy.
*
* @return A response with a too busy error.
*/
[[nodiscard]] Response
makeTooBusyError() const;
/**
* @brief Make a response when json parsing fails.
*
* @return A response with a json parsing error.
*/
[[nodiscard]] Response
makeJsonParsingError() const;
/**
* @beirf Compose an error into json object from a status.
*
* @param error The status to compose into a json object.
* @return The composed json object.
*/
[[nodiscard]] boost::json::object
composeError(rpc::Status const& error) const;
/**
* @brief Compose an error into json object from a rippled error.
*
* @param error The rippled error to compose into a json object.
* @return The composed json object.
*/
[[nodiscard]] boost::json::object
composeError(rpc::RippledError error) const;
};
} // namespace web::ng::impl

40
src/web/ng/impl/WsConnection.hpp
View File

@@ -28,6 +28,7 @@
#include "web/ng/Response.hpp"
#include "web/ng/impl/Concepts.hpp"
#include <boost/asio/buffer.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/ssl/context.hpp>
@@ -52,8 +53,20 @@
namespace web::ng::impl {
class WsConnectionBase : public Connection {
public:
using Connection::Connection;
virtual std::optional<Error>
sendBuffer(
boost::asio::const_buffer buffer,
boost::asio::yield_context yield,
std::chrono::steady_clock::duration timeout = Connection::DEFAULT_TIMEOUT
) = 0;
};
template <typename StreamType>
class WsConnection : public Connection {
class WsConnection : public WsConnectionBase {
boost::beast::websocket::stream<StreamType> stream_;
boost::beast::http::request<boost::beast::http::string_body> initialRequest_;
@@ -66,7 +79,7 @@ public:
util::TagDecoratorFactory const& tagDecoratorFactory
)
requires IsTcpStream<StreamType>
: Connection(std::move(ip), std::move(buffer), tagDecoratorFactory)
: WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
, stream_(std::move(socket))
, initialRequest_(std::move(initialRequest))
{
@@ -81,7 +94,7 @@ public:
util::TagDecoratorFactory const& tagDecoratorFactory
)
requires IsSslTcpStream<StreamType>
: Connection(std::move(ip), std::move(buffer), tagDecoratorFactory)
: WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
, stream_(std::move(socket), sslContext)
, initialRequest_(std::move(initialRequest))
{
@@ -111,6 +124,20 @@ public:
return true;
}
std::optional<Error>
sendBuffer(
boost::asio::const_buffer buffer,
boost::asio::yield_context yield,
std::chrono::steady_clock::duration timeout = Connection::DEFAULT_TIMEOUT
) override
{
auto error =
util::withTimeout([this, buffer](auto&& yield) { stream_.async_write(buffer, yield); }, yield, timeout);
if (error)
return error;
return std::nullopt;
}
std::optional<Error>
send(
Response response,
@@ -118,12 +145,7 @@ public:
std::chrono::steady_clock::duration timeout = DEFAULT_TIMEOUT
) override
{
auto error = util::withTimeout(
[this, &response](auto&& yield) { stream_.async_write(response.asConstBuffer(), yield); }, yield, timeout
);
if (error)
return error;
return std::nullopt;
return sendBuffer(response.asWsResponse(), yield, timeout);
}
std::expected<Request, Error>