'No Shared Cipher' Error with EDH-RSA-DES-CBC3-SHA

Question

I wanted to test TLS 1.0 connection with cipher EDH-RSA-DES-CBC3-SHA.

I test with openssl s_server and s_client. Works fine. Connection and data exchanges are fine.

openssl s_server -accept 4433 -cert server.pem -key serverkey.pem -cipher EDH-RSA-DES-CBC3-SHA -tls1

openssl s_client -connect 127.0.0.1:443 -cipher EDH-RSA-DES-CBC3-SHA -tls1

Shared ciphers:EDH-RSA-DES-CBC3-SHA

Now, I have another simple OpenSSL server code. With this and s_client the connection fails with server throwing the following:

3077613304:error:1408A0C1:SSL routines:SSL3_GET_CLIENT_HELLO:no shared cipher:s3_srvr.c:1361

I check that the libraries that s_server/s_client use and my server application use are same as follows:

cat /proc/9515/maps | awk '{print $6}' | grep '\.so' | sort | uniq | grep -e ssl -e libcrypto

/usr/lib/libcrypto.so.1.0.1e

/usr/lib/libssl.so.1.0.1e

However, for other ciphers like AES128-SHA, connection from s_client to my server application is fine.

Here is how I set up ctx in my server code:

SSL_CTX* InitServerCTX(void)
{
    SSL_CTX *ctx = NULL;
    SSL_library_init();
    OpenSSL_add_all_algorithms();
    SSL_load_error_strings(); 
    
    ctx = SSL_CTX_new(TLSv1_server_method());
    SSL_CTX_set_cipher_list(ctx,"EDH-RSA-DES-CBC3-SHA"); // Returns 1
    
    SSL_CTX_use_certificate_chain_file(ctx, "server.pem");
    SSL_CTX_use_PrivateKey_file(ctx, "serverkey.pem", SSL_FILETYPE_PEM);
    
    return ctx;
}

Why does my server application throw up 'No shared cipher' error and s_server is fine with same client?

Answer 1

@Cmidid's answer is correct. Here's a little more information, including the IETF DH groups and selecting a group size through the DH callback.

Its written in C++, but its easy enough to convert back to C.

using DH_ptr = std::unique_ptr<DH, decltype(&::DH_free)>;
using BIO_MEM_ptr = std::unique_ptr<BIO, decltype(&::BIO_free)>;
using BIO_FILE_ptr = std::unique_ptr<BIO, decltype(&::BIO_free)>;
using SSL_CTX_ptr = std::unique_ptr<SSL_CTX, decltype(&::SSL_CTX_free)>;

SSL_CTX* CreateServerContext(const string & domain)
{
    const SSL_METHOD* method = SSLv23_server_method();
    ASSERT(method != NULL);

    SSL_CTX_ptr t(SSL_CTX_new(method), ::SSL_CTX_free);
    ASSERT(t.get() != NULL);

    long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
            flags |= SSL_OP_NO_COMPRESSION;
            flags |= SSL_OP_SAFARI_ECDHE_ECDSA_BUG;
            flags |= SSL_OP_CIPHER_SERVER_PREFERENCE;

    SSL_CTX_set_options(t.get(), flags);

    string ciphers = "HIGH:!aNULL:!RC4:!MD5";
    rc = SSL_CTX_set_cipher_list(t.get(), ciphers.c_str());    
    ...

    LogDebug("GetServerContext: setting DH callback");
    SSL_CTX_set_tmp_dh_callback(t.get(), DhCallback);

    LogDebug("GetServerContext: setting ECDH callback");
    SSL_CTX_set_tmp_ecdh_callback(t.get(), EcdhCallback);
    ...

    return t.release();
}

DH* DhCallback(SSL *ssl, int is_export, int keylength)
{
    UNUSED(ssl);
    UNUSED(is_export);

#if defined(ALLOW_DH_1024_PARAMS)
    if (keylength <= 1024 + 4)
        return DH1024();
    else
#endif

    if (keylength <= 1536 + 4)
        return DH1536();
    else if (keylength <= 2048 + 4)
        return DH2048();
    else if (keylength <= 3072 + 4)
        return DH3072();
    else if (keylength <= 4096 + 4)
        return DH4096();

    return DH4096();
}

#if defined(ALLOW_DH_1024_PARAMS)
static DH* DH1024()
{
    static const char g_dh1024_sz[] = "-----BEGIN DH PARAMETERS-----\n"
    "MIGHAoGBAP//////////yQ/aoiFowjTExmKLgNwc0SkCTgiKZ8x0Agu+pjsTmyJR\n"
    "Sgh5jjQE3e+VGbPNOkMbMCsKbfJfFDdP4TVtbVHCReSFtXZiXn7G9ExC6aY37WsL\n"
    "/1y29Aa37e44a/taiZ+lrp8kEXxLH+ZJKGZR7OZTgf//////////AgEC\n"
    "-----END DH PARAMETERS-----";

    static DH_ptr dh(NULL, NULL);

    if (dh.get())
        return dh.get();

    BIO_MEM_ptr bio(BIO_new_mem_buf((void*) g_dh1024_sz, (int) sizeof(g_dh1024_sz)), ::BIO_free);
    ASSERT(bio.get());

    dh = DH_ptr(PEM_read_bio_DHparams(bio.get(), NULL, NULL, NULL), ::DH_free);
    unsigned long err = ERR_get_error();

    ASSERT(dh.get());
    return dh.get();
}
#endif

static DH* DH1536()
{
    static const char g_dh1536_sz[] = "-----BEGIN DH PARAMETERS-----\n"
            "MIHHAoHBAP//////////yQ/aoiFowjTExmKLgNwc0SkCTgiKZ8x0Agu+pjsTmyJR\n"
            "Sgh5jjQE3e+VGbPNOkMbMCsKbfJfFDdP4TVtbVHCReSFtXZiXn7G9ExC6aY37WsL\n"
            "/1y29Aa37e44a/taiZ+lrp8kEXxLH+ZJKGZR7ORbPcIAfLihY78FmNpINhxV05pp\n"
            "Fj+o/STPX4NlXSPco62WHGLzViCFUrue1SkHcJaWbWcMNU5KvJgE8XRsCMojcyf/\n"
            "/////////wIBAg==\n"
            "-----END DH PARAMETERS-----";

    static DH_ptr dh(NULL, NULL);

    if (dh.get())
        return dh.get();

    BIO_MEM_ptr bio(BIO_new_mem_buf((void*) g_dh1536_sz, (int) sizeof(g_dh1536_sz)), ::BIO_free);
    ASSERT(bio.get());

    dh = DH_ptr(PEM_read_bio_DHparams(bio.get(), NULL, NULL, NULL), ::DH_free);
    unsigned long err = ERR_get_error();

    ASSERT(dh.get());
    return dh.get();
}

static DH* DH2048()
{
    static const char g_dh2048_sz[] = "-----BEGIN DH PARAMETERS-----\n"
            "MIIBCAKCAQEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb\n"
            "IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft\n"
            "awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT\n"
            "mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh\n"
            "fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq\n"
            "5RXSJhiY+gUQFXKOWoqsqmj//////////wIBAg==\n"
            "-----END DH PARAMETERS-----";

    static DH_ptr dh(NULL, NULL);

    if (dh.get())
        return dh.get();

    BIO_MEM_ptr bio(BIO_new_mem_buf((void*) g_dh2048_sz, (int) sizeof(g_dh2048_sz)), ::BIO_free);
    ASSERT(bio.get());

    dh = DH_ptr(PEM_read_bio_DHparams(bio.get(), NULL, NULL, NULL), ::DH_free);
    unsigned long err = ERR_get_error();

    ASSERT(dh.get());
    return dh.get();
}

static DH* DH3072()
{
    static const char g_dh3072_sz[] = "-----BEGIN DH PARAMETERS-----\n"
            "MIIBiAKCAYEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb\n"
            "IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft\n"
            "awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT\n"
            "mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh\n"
            "fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq\n"
            "5RXSJhiY+gUQFXKOWoqqxC2tMxcNBFB6M6hVIavfHLpk7PuFBFjb7wqK6nFXXQYM\n"
            "fbOXD4Wm4eTHq/WujNsJM9cejJTgSiVhnc7j0iYa0u5r8S/6BtmKCGTYdgJzPshq\n"
            "ZFIfKxgXeyAMu+EXV3phXWx3CYjAutlG4gjiT6B05asxQ9tb/OD9EI5LgtEgqTrS\n"
            "yv//////////AgEC\n"
            "-----END DH PARAMETERS-----";

    static DH_ptr dh(NULL, NULL);

    if (dh.get())
        return dh.get();

    BIO_MEM_ptr bio(BIO_new_mem_buf((void*) g_dh3072_sz, (int) sizeof(g_dh3072_sz)), ::BIO_free);
    ASSERT(bio.get());

    dh = DH_ptr(PEM_read_bio_DHparams(bio.get(), NULL, NULL, NULL), ::DH_free);
    unsigned long err = ERR_get_error();

    ASSERT(dh.get());
    return dh.get();
}

static DH* DH4096()
{
    static const char g_dh4096_sz[] = "-----BEGIN DH PARAMETERS-----\n"
            "MIICCAKCAgEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb\n"
            "IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft\n"
            "awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT\n"
            "mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh\n"
            "fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq\n"
            "5RXSJhiY+gUQFXKOWoqqxC2tMxcNBFB6M6hVIavfHLpk7PuFBFjb7wqK6nFXXQYM\n"
            "fbOXD4Wm4eTHq/WujNsJM9cejJTgSiVhnc7j0iYa0u5r8S/6BtmKCGTYdgJzPshq\n"
            "ZFIfKxgXeyAMu+EXV3phXWx3CYjAutlG4gjiT6B05asxQ9tb/OD9EI5LgtEgqSEI\n"
            "ARpyPBKnh+bXiHGaEL26WyaZwycYavTiPBqUaDS2FQvaJYPpyirUTOjbu8LbBN6O\n"
            "+S6O/BQfvsqmKHxZR05rwF2ZspZPoJDDoiM7oYZRW+ftH2EpcM7i16+4G912IXBI\n"
            "HNAGkSfVsFqpk7TqmI2P3cGG/7fckKbAj030Nck0BjGZ//////////8CAQI=\n"
            "-----END DH PARAMETERS-----";

    static DH_ptr dh(NULL, NULL);

    if (dh.get())
        return dh.get();

    BIO_MEM_ptr bio(BIO_new_mem_buf((void*) g_dh4096_sz, (int) sizeof(g_dh4096_sz)), ::BIO_free);
    ASSERT(bio.get());

    dh = DH_ptr(PEM_read_bio_DHparams(bio.get(), NULL, NULL, NULL), ::DH_free);
    unsigned long err = ERR_get_error();

    ASSERT(dh.get());
    return dh.get();
}

Answer 2

You need to create a DH object and set up the DH parameters for the ssl context ctx. To be more specific setting the primp p and generator g is required once the DH object is allocated.

One way to do that would be to use below example pseudo code

Here dh512_p dh512_g are the primo p and generator g respectively

DH* get_dh512(const unsigned char *dh512_p,const unsigned char *dh512_g)
{
    DH *dh=NULL;
    if ((dh=DH_new()) == NULL) return(NULL);
    dh->p=BN_bin2bn(dh512_p,sizeof(dh512_p),NULL);
    dh->g=BN_bin2bn(dh512_g,sizeof(dh512_g),NULL);
    if ((dh->p == NULL) || (dh->g == NULL))
        return(NULL);

   return(dh);
}

Then set up the parameters in your function using the callback

//if key exchange is based on diffie hellman
DH *dh =  get_dh512(dh512_p,dh512_g)
SSL_CTX_set_tmp_dh(ctx,dh);
SSL_CTX_set_cipher_list(ctx,ciphers);

Please look at the following links for callback details http://linux.die.net/man/3/ssl_ctx_set_tmp_dh