ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

New http::client_session for making requests:

Browse Source 
- Add 10,000 popular URL data set for unit tests
This commit is contained in:
Vinnie Falco
2014-03-19 09:28:19 -07:00
parent 6e8230e1fb
commit bfa8dec6f6
16 changed files with 11831 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

383
beast/http/tests/client_session.test.cpp Normal file
View File

@@ -0,0 +1,383 @@
//------------------------------------------------------------------------------
/*
This file is part of Beast: https://github.com/vinniefalco/Beast
Copyright 2013, Vinnie Falco <vinnie.falco@gmail.com>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
// LIBS: pthread
// MODULES: urls_large_data.cpp ../impl/raw_parser.cpp ../impl/joyent_parser.cpp
#if BEAST_INCLUDE_BEASTCONFIG
#include "../../BeastConfig.h"
#endif
#include "../../unit_test/suite.h"
#include "urls_large_data.h"
#include "../client_session.h"
#include "../get.h"
#include "../../asio/bind_handler.h"
#include "../../asio/memory_buffer.h"
#include "../../utility/ci_char_traits.h"
#include <boost/asio.hpp>
#include <thread>
#include <unordered_map>
namespace beast {
namespace http {
/** Allows thread-safe forward traversal of a sequence.
Each time the shared_iterator is dereferenced it provides an element in
the sequence or the one-past-the-end iterator if there are no elements
remaining in the sequence. Access to the shared iterator is thread safe:
multiple threads of execution can request iterators from the sequence,
and no two threads will see the same iterator.
Any operations on the underlying container which would invalidate
iterators or change the sequence of elements pointed to by the range
of iterators referenced by the shared_iterator, results in undefined
behavior.
*/
template <class Iterator>
class shared_iterator
{
public:
static_assert (std::is_same <Iterator, std::decay_t <Iterator>>::value,
"Iterator may not be a reference or const type");
typedef Iterator value_type;
private:
std::mutex m_mutex;
Iterator m_iter;
Iterator m_end;
public:
/** Construct the iteration from the range [first, last) */
shared_iterator (Iterator first, Iterator last)
: m_iter (first)
, m_end (last)
{
}
/** Obtains the next iterator in the sequence.
Post-condition
Current shared position in the sequence is advanced by one.
Thread safety:
Can be called from any thread at any time.
*/
Iterator
operator* ()
{
std::lock_guard <decltype(m_mutex)> lock (m_mutex);
if (m_iter == m_end)
return m_iter;
return m_iter++;
}
/** Returns the one-past-the end iterator for the sequence.
Thread safety:
Can be called from any thread at any time.
*/
Iterator
end() const
{
return m_end;
}
};
//------------------------------------------------------------------------------
class client_session_test : public unit_test::suite
{
public:
typedef boost::system::error_code error_code;
//--------------------------------------------------------------------------
/** Used to submit HTTP requests. */
class Request
{
private:
typedef std::string value_type;
typedef std::string string_type;
std::unordered_map <std::string, string_type> m_headers;
// This also works, for allowing header values to
// span multiple discontiguous memory buffers.
//
std::unordered_map <std::string,
std::vector <std::string>> m_headers_plus;
class vector_proxy
{
private:
std::reference_wrapper <std::vector <string_type>> m_vec;
public:
explicit vector_proxy (std::vector <string_type>& vec)
: m_vec (vec)
{
}
vector_proxy& operator= (string_type const& s)
{
m_vec.get().emplace_back (s);
return *this;
}
};
public:
bool keep_alive()
{
return false;
}
// Use this to set the fields
std::string&
operator[] (std::string const& field)
{
return m_headers[field];
}
/** Calls Function for each header.
Requirements:
`X` The type `Function`
`Y` A type meeting this requirement:
ConvertibleToConstBuffer
`Z` A type meeting either requirement:
ConstBufferSequence
ConvertibleToConstBuffer
`f` A value of type `X`
`n` A value of type `Y`
`v` A value of type `Z`
The expression
f (n, v);
must be well formed.
*/
template <class Function>
void
headers (Function f)
{
for (auto const& h : m_headers)
f (h.first, h.second);
}
};
//--------------------------------------------------------------------------
class Response
{
private:
typedef boost::system::error_code error_code;
asio::memory_buffer <std::uint8_t> m_buffer;
boost::asio::streambuf m_body;
public:
enum
{
buffer_bytes = 4192
};
typedef std::vector <std::pair <
std::string, std::string>> headers_type;
headers_type headers;
Response()
: m_buffer (buffer_bytes)
{
}
boost::asio::mutable_buffer
buffer()
{
return boost::asio::mutable_buffer (
m_buffer.data(), m_buffer.size());
}
template <class FieldString, class ValueString>
error_code
header (FieldString const& field, ValueString const& value)
{
headers.push_back (std::make_pair (field, value));
return error_code();
}
error_code
body (boost::asio::const_buffer in)
{
m_body.commit (boost::asio::buffer_copy (
m_body.prepare (boost::asio::buffer_size (in)),
boost::asio::buffer (in)));
return error_code();
}
std::size_t
size() const
{
return m_body.size();
}
std::string
data() const
{
std::string s;
s.resize (m_body.size());
boost::asio::buffer_copy (boost::asio::buffer (
&s[0], s.size()), m_body.data());
return s;
}
};
//--------------------------------------------------------------------------
template <class Session>
error_code
visit (Session& session, std::string const& url)
{
error_code ec;
typedef boost::asio::ip::tcp::resolver resolver_t;
boost::asio::io_service ios;
resolver_t r (ios);
auto iter (r.resolve (resolver_t::query (
url, "80", resolver_t::query::numeric_service), ec));
if (ec)
return ec;
if (iter != resolver_t::iterator())
{
session.next_layer().connect (iter->endpoint(), ec);
if (ec)
return ec;
Request req;
req ["User-Agent"] = "rippled-http-client/1.0";
req ["Host"] = url + ":80";
req ["Content-Type"] = "application/text";
req ["Accept"] = "application/text";
//req ["Content-length"] = "0";
//req.prepare ("GET / HTTP/1.0");
Response resp;
ec = session.get (req, resp);
if (ec)
{
// hack
session.next_layer().close();
}
log <<
"GET " << url << " " << ec.message();
for (auto const& h : resp.headers)
log << h.first << ": " << h.second;
log << resp.data();
log << " ";
}
return ec;
}
//--------------------------------------------------------------------------
template <class Iterator>
void
concurrent_get (shared_iterator <Iterator>& iter)
{
typedef boost::asio::ip::tcp::socket socket_type;
for (auto cur (*iter); cur != iter.end(); cur = *iter)
{
sync_client_session <socket_type> session;
std::string const base (*cur);
std::string url;
url = "www." + base;
auto const ec (visit (session, url));
}
}
// Perform HTTP get on a sequence of URLs in parallel
// Requirements
// Sequence must me
// Sequence::value_type must be convertible to std::string
template <class Iterator>
void test_concurrent_get (Iterator first, Iterator last)
{
#if 0
last = first;
std::advance (last, 3000);
#endif
shared_iterator <Iterator> iter (first, last);
std::vector <std::thread> pool;
#if 0
std::size_t const hardware_concurrency (
std::max (std::thread::hardware_concurrency(),
2u
));
#else
std::size_t const hardware_concurrency (1);
#endif
for (std::size_t n (hardware_concurrency); n--;)
pool.emplace_back (std::bind (
&client_session_test::concurrent_get <Iterator>, this,
std::ref (iter)));
for (auto& t : pool)
t.join();
pass();
}
template <class Sequence>
void test_concurrent_get (Sequence const& sequence)
{
auto last (std::begin(sequence));
std::advance (last, std::min (std::size_t(1), sequence.size()));
test_concurrent_get (std::begin (sequence), last);
}
void test_get()
{
get ("http://www.google.com");
}
void run()
{
//test_get();
test_concurrent_get (urls_large_data());
}
};
BEAST_DEFINE_TESTSUITE_MANUAL(client_session,http,beast);
}
}