The test consists of a server and a client sending messages of a fixed size back and forth, repeating for a long time. They are both single-threaded and written in C++ using the Boost ASIO library. Communication is local and is over TCP with TLS. I am monitoring the loopback interface using bmon, and checking CPU usage with System Monitor.
When I use a message size of 16384, I see RX/TX speeds of about 200 MiB/s, and CPU usage is about 60%. When I change the message size to 16385, RX/TX drops to about 300 KiB/s, and CPU usage drops to some small percentage (<10%).
I would like to know the reason for this tremendous drop in throughput? I suspect it has something to do with TLS, since there was no drop when using plain TCP. However, a drop of 2000:3 seems pretty drastic, especially considering that the program doesn't seem to be CPU bound either.
My current guess is that 16384 is a hardcoded/configured limit (maybe TLS max record size?), and exceeding this number requires extra messaging, but why wouldn't the throughput drop by ~2:1 instead of 2000:3? Can anyone help explain this?
Here is the complete example code:
Server.cpp
#include <functional>
#include <iostream>
#include <memory>
#include <system_error>
#include "asio.hpp"
#include "asio/ssl.hpp"
#define MESSAGE_SIZE 16385
class Server
{
public:
struct Certs
{
std::string certificate_chain_file;
std::string private_key_file;
std::string verify_file;
};
using Stream = asio::ssl::stream<asio::ip::tcp::socket>;
Server(Certs certs, unsigned short port)
: acceptor(io_context, asio::ip::tcp::endpoint(asio::ip::tcp::v4(), port)),
ssl_context(asio::ssl::context::tlsv12),
rw_buf(new uint8_t[MESSAGE_SIZE])
{
ssl_context.set_options(
asio::ssl::context::default_workarounds |
asio::ssl::context::no_sslv2 |
asio::ssl::context::no_sslv3 |
asio::ssl::context::no_tlsv1 |
asio::ssl::context::no_tlsv1_1 |
asio::ssl::context::single_dh_use);
ssl_context.use_certificate_chain_file(certs.certificate_chain_file);
ssl_context.use_private_key_file(certs.private_key_file, asio::ssl::context::pem);
ssl_context.set_verify_mode(
asio::ssl::context::verify_peer |
asio::ssl::context::verify_fail_if_no_peer_cert);
ssl_context.load_verify_file(certs.verify_file);
acceptor.async_accept(std::bind(&Server::onAccept, this, std::placeholders::_1, std::placeholders::_2));
}
void run()
{
io_context.run();
}
void onAccept(const std::error_code& error, asio::ip::tcp::socket socket)
{
if (error)
{
std::cerr << "Accept error=" << error.message() << std::endl;
return;
}
stream.reset(new Stream(std::move(socket), ssl_context));
asyncHandshake();
}
void asyncHandshake()
{
stream->async_handshake(
asio::ssl::stream_base::server,
std::bind(&Server::onHandshake, this, std::placeholders::_1));
}
void onHandshake(const std::error_code& error)
{
if (error)
{
std::cerr << "Handshake error=" << error.message() << std::endl;
return;
}
asyncReadMessage();
}
void asyncReadMessage()
{
asio::async_read(
*stream,
asio::buffer(rw_buf.get(), MESSAGE_SIZE),
std::bind(&Server::onRead, this, std::placeholders::_1, std::placeholders::_2));
}
void asyncWriteMessage()
{
asio::async_write(
*stream,
asio::buffer(rw_buf.get(), MESSAGE_SIZE),
std::bind(&Server::onWrite, this, std::placeholders::_1, std::placeholders::_2));
}
void onRead(const std::error_code& error, size_t bytes_transferred)
{
if (error)
{
std::cerr << "Read error=" << error.message() << std::endl;
return;
}
asyncWriteMessage();
}
void onWrite(const std::error_code& error, size_t bytes_transferred)
{
if (error)
{
std::cerr << "Write error=" << error.message() << std::endl;
return;
}
asyncReadMessage();
}
protected:
asio::io_context io_context;
asio::ip::tcp::acceptor acceptor;
asio::ssl::context ssl_context;
std::unique_ptr<Stream> stream;
std::unique_ptr<uint8_t[]> rw_buf;
};
int main(int argc, char* argv[])
{
Server::Certs certs
{
.certificate_chain_file = "server.crt",
.private_key_file = "server.key",
.verify_file = "ca.crt"
};
unsigned short port = 9090;
Server server(certs, port);
server.run();
}
Client.cpp
#include <functional>
#include <iostream>
#include <memory>
#include <system_error>
#include "asio.hpp"
#include "asio/ssl.hpp"
#define MESSAGE_SIZE 16385
#define MESSAGE_BURST_SIZE 50000
class Client
{
public:
struct Certs
{
std::string certificate_chain_file;
std::string private_key_file;
std::string verify_file;
};
using Stream = asio::ssl::stream<asio::ip::tcp::socket>;
Client(Certs certs)
: ssl_context(asio::ssl::context::tlsv12),
rw_buf(new uint8_t[MESSAGE_SIZE])
{
ssl_context.set_options(
asio::ssl::context::default_workarounds |
asio::ssl::context::no_sslv2 |
asio::ssl::context::no_sslv3 |
asio::ssl::context::no_tlsv1 |
asio::ssl::context::no_tlsv1_1 |
asio::ssl::context::single_dh_use);
ssl_context.use_certificate_chain_file(certs.certificate_chain_file);
ssl_context.use_private_key_file(certs.private_key_file, asio::ssl::context::pem);
ssl_context.set_verify_mode(
asio::ssl::context::verify_peer |
asio::ssl::context::verify_fail_if_no_peer_cert);
ssl_context.load_verify_file(certs.verify_file);
}
void run(std::string host, unsigned short port)
{
stream.reset(new Stream(std::move(asio::ip::tcp::socket(io_context)), ssl_context));
asio::ip::tcp::endpoint endpoint(asio::ip::address::from_string(host), port);
stream->lowest_layer().async_connect(endpoint, std::bind(&Client::onConnect, this, std::placeholders::_1));
io_context.run();
}
void onConnect(const std::error_code& error)
{
if (error)
{
std::cerr << "Connect error=" << error.message() << std::endl;
return;
}
asyncHandshake();
}
void asyncHandshake()
{
stream->async_handshake(
asio::ssl::stream_base::client,
std::bind(&Client::onHandshake, this, std::placeholders::_1));
}
void onHandshake(const std::error_code& error)
{
if (error)
{
std::cerr << "Handshake error=" << error.message() << std::endl;
return;
}
asyncWriteMessage();
}
void asyncReadMessage()
{
asio::async_read(
*stream,
asio::buffer(rw_buf.get(), MESSAGE_SIZE),
std::bind(&Client::onRead, this, std::placeholders::_1, std::placeholders::_2));
}
void asyncWriteMessage()
{
asio::async_write(
*stream,
asio::buffer(rw_buf.get(), MESSAGE_SIZE),
std::bind(&Client::onWrite, this, std::placeholders::_1, std::placeholders::_2));
}
void onRead(const std::error_code& error, size_t bytes_transferred)
{
if (error)
{
std::cerr << "Read error=" << error.message() << std::endl;
return;
}
if (++message_count >= MESSAGE_BURST_SIZE)
{
return;
}
asyncWriteMessage();
}
void onWrite(const std::error_code& error, size_t bytes_transferred)
{
if (error)
{
std::cerr << "Write error=" << error.message() << std::endl;
return;
}
asyncReadMessage();
}
protected:
asio::io_context io_context;
asio::ssl::context ssl_context;
std::unique_ptr<Stream> stream;
std::unique_ptr<uint8_t[]> rw_buf;
int message_count = 0;
};
int main(int argc, char* argv[])
{
Client::Certs certs
{
.certificate_chain_file = "client.crt",
.private_key_file = "client.key",
.verify_file = "ca.crt"
};
std::string host = "127.0.0.1";
unsigned short port = 9090;
Client client(certs);
client.run(host, port);
}
I remember reading a story¹ about debugging this on extensive network infrastructure.
The TL;DR of it was: Jumbo Frames.
Jumbo Frames are not equally supported by all parts of the network trajectory, and when it fails, it takes some time for the error to be detected, after which the transmission will have to be retried with smaller sizes.
In some circumstances the retrying/back-off was worst-case leading to highly reduced throughput.
So, tuning the max frame sizes (MTU) on the relevant network equipment to some "safish common denominator" might help. Also, trying to exert finer control over packet sizes on the wire could help (I think disabling Nagle's algorithm usually makes an appearance here. Be sure you absolutely fully understand your traffic patterns before you do, of course).
¹ not my story, finding a link in due time
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With