Can you help me get my head around openssl public key encryption with rsa.h in c++?

Question

I am trying to get my head around public key encryption using the openssl implementation of rsa in C++. Can you help? So far these are my thoughts (please do correct if necessary)

1. Alice is connected to Bob over a network
2. Alice and Bob want secure communications
3. Alice generates a public / private key pair and sends public key to Bob
4. Bob receives public key and encrypts a randomly generated symmetric cypher key (e.g. blowfish) with the public key and sends the result to Alice
5. Alice decrypts the ciphertext with the originally generated private key and obtains the symmetric blowfish key
6. Alice and Bob now both have knowledge of symmetric blowfish key and can establish a secure communication channel

Now, I have looked at the openssl/rsa.h rsa implementation (since I already have practical experience with openssl/blowfish.h), and I see these two functions:

int RSA_public_encrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa, int padding);
int RSA_private_decrypt(int flen, unsigned char *from,
 unsigned char *to, RSA *rsa, int padding);

If Alice is to generate *rsa, how does this yield the rsa key pair? Is there something like rsa_public and rsa_private which are derived from rsa? Does *rsa contain both public and private key and the above function automatically strips out the necessary key depending on whether it requires the public or private part? Should two unique *rsa pointers be generated so that actually, we have the following:

int RSA_public_encrypt(int flen, unsigned char *from,
unsigned char *to, RSA *rsa_public, int padding);
int RSA_private_decrypt(int flen, unsigned char *from,
 unsigned char *to, RSA *rsa_private, int padding);

Secondly, in what format should the *rsa public key be sent to Bob? Must it be reinterpreted in to a character array and then sent the standard way? I've heard something about certificates -- are they anything to do with it?

Sorry for all the questions, Best Wishes, Ben.

EDIT: Coe I am currently employing:

/*
 *  theEncryptor.cpp
 *  
 *
 *  Created by ben on 14/01/2010.
 *  Copyright 2010 __MyCompanyName__. All rights reserved.
 *
 */

#include "theEncryptor.h"
#include <iostream>
#include <sys/socket.h>
#include <sstream>

theEncryptor::theEncryptor()
{

}

void
theEncryptor::blowfish(unsigned char *data, int data_len, unsigned char* key, int enc)
{

    //  hash the key first! 
    unsigned char obuf[20];
    bzero(obuf,20);
    SHA1((const unsigned char*)key, 64, obuf);

    BF_KEY bfkey;
    int keySize = 16;//strlen((char*)key);
    BF_set_key(&bfkey, keySize, obuf);

    unsigned char ivec[16];
    memset(ivec, 0, 16);

    unsigned char* out=(unsigned char*) malloc(data_len);
    bzero(out,data_len);
    int num = 0;
    BF_cfb64_encrypt(data, out, data_len, &bfkey, ivec, &num, enc);

    //for(int i = 0;i<data_len;i++)data[i]=out[i];

    memcpy(data, out, data_len);
    free(out);  

}

void
theEncryptor::generateRSAKeyPair(int bits)
{
    rsa = RSA_generate_key(bits, 65537, NULL, NULL);
}


int
theEncryptor::publicEncrypt(unsigned char* data, unsigned char* dataEncrypted,int dataLen)
{   
    return RSA_public_encrypt(dataLen, data, dataEncrypted, rsa, RSA_PKCS1_OAEP_PADDING);   
}

int
theEncryptor::privateDecrypt(unsigned char* dataEncrypted,
                             unsigned char* dataDecrypted)
{
    return RSA_private_decrypt(RSA_size(rsa), dataEncrypted, 
                                   dataDecrypted, rsa, RSA_PKCS1_OAEP_PADDING);
}

void 
theEncryptor::receivePublicKeyAndSetRSA(int sock, int bits)
{
    int max_hex_size = (bits / 4) + 1;
    char keybufA[max_hex_size];
    bzero(keybufA,max_hex_size);
    char keybufB[max_hex_size];
    bzero(keybufB,max_hex_size);
    int n = recv(sock,keybufA,max_hex_size,0); 
    n = send(sock,"OK",2,0);
    n = recv(sock,keybufB,max_hex_size,0); 
    n = send(sock,"OK",2,0); 
    rsa = RSA_new();
    BN_hex2bn(&rsa->n, keybufA);
    BN_hex2bn(&rsa->e, keybufB);
}

void 
theEncryptor::transmitPublicKey(int sock, int bits)
{
    const int max_hex_size = (bits / 4) + 1;
    long size = max_hex_size;
    char keyBufferA[size];
    char keyBufferB[size];
    bzero(keyBufferA,size);
    bzero(keyBufferB,size);
    sprintf(keyBufferA,"%s\r\n",BN_bn2hex(rsa->n));
    sprintf(keyBufferB,"%s\r\n",BN_bn2hex(rsa->e));
    int n = send(sock,keyBufferA,size,0);
    char recBuf[2];
    n = recv(sock,recBuf,2,0);
    n = send(sock,keyBufferB,size,0);
    n = recv(sock,recBuf,2,0);
}

void
theEncryptor::generateRandomBlowfishKey(unsigned char* key, int bytes)
{
            /*
    srand( (unsigned)time( NULL ) );
    std::ostringstream stm;
    for(int i = 0;i<bytes;i++){
        int randomValue = 65 + rand()% 26;
        stm << (char)((int)randomValue);
    }
    std::string str(stm.str());
    const char* strs = str.c_str();
    for(int i = 0;bytes;i++)key[i]=strs[i];
            */

    int n = RAND_bytes(key, bytes);

    if(n==0)std::cout<<"Warning key was generated with bad entropy. You should not consider communication to be secure"<<std::endl;

}

theEncryptor::~theEncryptor(){}

Answer 1

You should actually be using the higher-level "Envelope Encryption" functions from openssl/evp.h, rather than the low-level RSA functions directly. These do most of the work for you and mean you don't have to reinvent the wheel.

In this case, you'd use the EVP_SealInit(), EVP_SealUpdate() and EVP_SealFinal() functions. The corresponding decryption functions are EVP_OpenInit(), EVP_OpenUpdate() and EVP_OpenFinal(). I would suggest using EVP_aes_128_cbc() as the value of the cipher type parameter.

Once you've got the public key loaded into an RSA * handle, you use EVP_PKEY_assign_RSA() to put it into an EVP_PKEY * handle for the EVP functions.

Once you've got this going, to solve the authentication problem I mentioned in my comment, you'll need to established a trusted authority ("Trent"). Trent's public key is known to all users (distributed with the application or similar - just load it from a PEM file). Instead of exchanging bare RSA parameters, Alice and Bob exchange x509 certificates that contain their RSA public keys together with their name, and are signed by Trent. Alice and Bob then each verify the certificate they recieved from the other (using Trent's public key, which they already know), including checking that the associated name is the right one, before continuing the protocol. OpenSSL includes functions for loading and verifying certificates in the x509.h header.

---

Here's an example of how to use EVP_Seal*() to encrypt a file given the recipient's public key. It takes the PEM RSA Public Key file (ie as generated by openssl rsa -pubout) as a command line argument, reads the source data from stdin and writes the encrypted data to stdout. To decrypt, use EVP_Open*() instead, and PEM_read_RSAPrivateKey() to read a private key rather than public key.

It's not really that hard - and certainly less error prone than messing about generating padding, IVs and so on yourself (the Seal function does both the RSA and AES parts of the deal). Anyway, the code:

#include <stdio.h>
#include <stdlib.h>

#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/err.h>

#include <arpa/inet.h> /* For htonl() */

int do_evp_seal(FILE *rsa_pkey_file, FILE *in_file, FILE *out_file)
{
    int retval = 0;
    RSA *rsa_pkey = NULL;
    EVP_PKEY *pkey = EVP_PKEY_new();
    EVP_CIPHER_CTX ctx;
    unsigned char buffer[4096];
    unsigned char buffer_out[4096 + EVP_MAX_IV_LENGTH];
    size_t len;
    int len_out;
    unsigned char *ek;
    int eklen;
    uint32_t eklen_n;
    unsigned char iv[EVP_MAX_IV_LENGTH];

    if (!PEM_read_RSA_PUBKEY(rsa_pkey_file, &rsa_pkey, NULL, NULL))
    {
        fprintf(stderr, "Error loading RSA Public Key File.\n");
        ERR_print_errors_fp(stderr);
        retval = 2;
        goto out;
    }

    if (!EVP_PKEY_assign_RSA(pkey, rsa_pkey))
    {
        fprintf(stderr, "EVP_PKEY_assign_RSA: failed.\n");
        retval = 3;
        goto out;
    }

    EVP_CIPHER_CTX_init(&ctx);
    ek = malloc(EVP_PKEY_size(pkey));

    if (!EVP_SealInit(&ctx, EVP_aes_128_cbc(), &ek, &eklen, iv, &pkey, 1))
    {
        fprintf(stderr, "EVP_SealInit: failed.\n");
        retval = 3;
        goto out_free;
    }

    /* First we write out the encrypted key length, then the encrypted key,
     * then the iv (the IV length is fixed by the cipher we have chosen).
     */

    eklen_n = htonl(eklen);
    if (fwrite(&eklen_n, sizeof eklen_n, 1, out_file) != 1)
    {
        perror("output file");
        retval = 5;
        goto out_free;
    }
    if (fwrite(ek, eklen, 1, out_file) != 1)
    {
        perror("output file");
        retval = 5;
        goto out_free;
    }
    if (fwrite(iv, EVP_CIPHER_iv_length(EVP_aes_128_cbc()), 1, out_file) != 1)
    {
        perror("output file");
        retval = 5;
        goto out_free;
    }

    /* Now we process the input file and write the encrypted data to the
     * output file. */

    while ((len = fread(buffer, 1, sizeof buffer, in_file)) > 0)
    {
        if (!EVP_SealUpdate(&ctx, buffer_out, &len_out, buffer, len))
        {
            fprintf(stderr, "EVP_SealUpdate: failed.\n");
            retval = 3;
            goto out_free;
        }

        if (fwrite(buffer_out, len_out, 1, out_file) != 1)
        {
            perror("output file");
            retval = 5;
            goto out_free;
        }
    }

    if (ferror(in_file))
    {
        perror("input file");
        retval = 4;
        goto out_free;
    }

    if (!EVP_SealFinal(&ctx, buffer_out, &len_out))
    {
        fprintf(stderr, "EVP_SealFinal: failed.\n");
        retval = 3;
        goto out_free;
    }

    if (fwrite(buffer_out, len_out, 1, out_file) != 1)
    {
        perror("output file");
        retval = 5;
        goto out_free;
    }

    out_free:
    EVP_PKEY_free(pkey);
    free(ek);

    out:
    return retval;
}

int main(int argc, char *argv[])
{
    FILE *rsa_pkey_file;
    int rv;

    if (argc < 2)
    {
        fprintf(stderr, "Usage: %s <PEM RSA Public Key File>\n", argv[0]);
        exit(1);
    }

    rsa_pkey_file = fopen(argv[1], "rb");
    if (!rsa_pkey_file)
    {
        perror(argv[1]);
        fprintf(stderr, "Error loading PEM RSA Public Key File.\n");
        exit(2);
    }

    rv = do_evp_seal(rsa_pkey_file, stdin, stdout);

    fclose(rsa_pkey_file);
    return rv;
}

---

The code you've posted illustrates nicely why you should use the higher-level functions - you've fallen into a couple of pitfalls:

• rand() is emphatically not a cryptographically strong random number generator! Generating your symmetric key using rand() is enough to make the entire system completely insecure. (The EVP_*() functions generate the necessary random numbers themselves, using a cryptographically strong RNG, seeded from an appropriate entropy source).

• You are setting the IV for CFB mode to a fixed value (zero). This negates any advantage of using CFB mode in the first place (allowing attackers to trivially perform block-replacement attacks and worse). (The EVP_*() functions generate an appropriate IV for you, when required).

• RSA_PKCS1_OAEP_PADDING should be used if you're defining a new protocol, rather than interoperating with an existing protocol.

---

The corresponding decryption code, for posterity:

#include <stdio.h>
#include <stdlib.h>

#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/err.h>

#include <arpa/inet.h> /* For htonl() */

int do_evp_unseal(FILE *rsa_pkey_file, FILE *in_file, FILE *out_file)
{
    int retval = 0;
    RSA *rsa_pkey = NULL;
    EVP_PKEY *pkey = EVP_PKEY_new();
    EVP_CIPHER_CTX ctx;
    unsigned char buffer[4096];
    unsigned char buffer_out[4096 + EVP_MAX_IV_LENGTH];
    size_t len;
    int len_out;
    unsigned char *ek;
    unsigned int eklen;
    uint32_t eklen_n;
    unsigned char iv[EVP_MAX_IV_LENGTH];

    if (!PEM_read_RSAPrivateKey(rsa_pkey_file, &rsa_pkey, NULL, NULL))
    {
        fprintf(stderr, "Error loading RSA Private Key File.\n");
        ERR_print_errors_fp(stderr);
        retval = 2;
        goto out;
    }

    if (!EVP_PKEY_assign_RSA(pkey, rsa_pkey))
    {
        fprintf(stderr, "EVP_PKEY_assign_RSA: failed.\n");
        retval = 3;
        goto out;
    }

    EVP_CIPHER_CTX_init(&ctx);
    ek = malloc(EVP_PKEY_size(pkey));

    /* First need to fetch the encrypted key length, encrypted key and IV */

    if (fread(&eklen_n, sizeof eklen_n, 1, in_file) != 1)
    {
        perror("input file");
        retval = 4;
        goto out_free;
    }
    eklen = ntohl(eklen_n);
    if (eklen > EVP_PKEY_size(pkey))
    {
        fprintf(stderr, "Bad encrypted key length (%u > %d)\n", eklen,
            EVP_PKEY_size(pkey));
        retval = 4;
        goto out_free;
    }
    if (fread(ek, eklen, 1, in_file) != 1)
    {
        perror("input file");
        retval = 4;
        goto out_free;
    }
    if (fread(iv, EVP_CIPHER_iv_length(EVP_aes_128_cbc()), 1, in_file) != 1)
    {
        perror("input file");
        retval = 4;
        goto out_free;
    }

    if (!EVP_OpenInit(&ctx, EVP_aes_128_cbc(), ek, eklen, iv, pkey))
    {
        fprintf(stderr, "EVP_OpenInit: failed.\n");
        retval = 3;
        goto out_free;
    }

    while ((len = fread(buffer, 1, sizeof buffer, in_file)) > 0)
    {
        if (!EVP_OpenUpdate(&ctx, buffer_out, &len_out, buffer, len))
        {
            fprintf(stderr, "EVP_OpenUpdate: failed.\n");
            retval = 3;
            goto out_free;
        }

        if (fwrite(buffer_out, len_out, 1, out_file) != 1)
        {
            perror("output file");
            retval = 5;
            goto out_free;
        }
    }

    if (ferror(in_file))
    {
        perror("input file");
        retval = 4;
        goto out_free;
    }

    if (!EVP_OpenFinal(&ctx, buffer_out, &len_out))
    {
        fprintf(stderr, "EVP_OpenFinal: failed.\n");
        retval = 3;
        goto out_free;
    }

    if (fwrite(buffer_out, len_out, 1, out_file) != 1)
    {
        perror("output file");
        retval = 5;
        goto out_free;
    }

    out_free:
    EVP_PKEY_free(pkey);
    free(ek);

    out:
    return retval;
}

int main(int argc, char *argv[])
{
    FILE *rsa_pkey_file;
    int rv;

    if (argc < 2)
    {
        fprintf(stderr, "Usage: %s <PEM RSA Private Key File>\n", argv[0]);
        exit(1);
    }

    rsa_pkey_file = fopen(argv[1], "rb");
    if (!rsa_pkey_file)
    {
        perror(argv[1]);
        fprintf(stderr, "Error loading PEM RSA Private Key File.\n");
        exit(2);
    }

    rv = do_evp_unseal(rsa_pkey_file, stdin, stdout);

    fclose(rsa_pkey_file);
    return rv;
}