Encryption and decryption error 0x0407106B using OpenSSL

Question

I'm writing a routine in C that reads a base64 string with the public key and proceeds to encrypt a string. I also test the same string's decryption but I'm getting error 0x0407106B when trying to do the decoding:

$ openssl errstr 0x0407106B
error:0407106B:rsa routines:RSA_padding_check_PKCS1_type_2:block type is not 02

Here's the code

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include <openssl/rsa.h>
#include <openssl/engine.h>

//#define PADDING RSA_PKCS1_OAEP_PADDING
#define PADDING RSA_PKCS1_PADDING
//#define PADDING RSA_NO_PADDING

main() {

// public key
char *b64_pKey = "-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCp2w+8HUdECo8V5yuKYrWJmUbL\ntD6nSyVifN543axXvNSFzQfWNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6\nhsZA81AblAOOXKaUaxvFC+ZKRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68Epn\naOLepTZw+GLTnusQgwIDAQAB\n-----END PUBLIC KEY-----\n";

// private key
char *b64priv_key = "-----BEGIN RSA PRIVATE KEY-----\nMIICXAIBAAKBgQCp2w+8HUdECo8V5yuKYrWJmUbLtD6nSyVifN543axXvNSFzQfW\nNOGVkMsCo6W4hpl5eHv1p9Hqdcf/ZYQDWCK726u6hsZA81AblAOOXKaUaxvFC+ZK\nRJf+MtUGnv0v7CrGoblm1mMC/OQI1JfSsYi68EpnaOLepTZw+GLTnusQgwIDAQAB\nAoGBAKDuq3PikblH/9YS11AgwjwC++7ZcltzeZJdGTSPY1El2n6Dip9ML0hUjeSM\nROIWtac/nsNcJCnvOnUjK/c3NIAaGJcfRPiH/S0Ga6ROiDfFj2UXAmk/v4wRRUzr\n5lsA0jgEt5qcq2Xr/JPQVGB4wUgL/yQK0dDhW0EdrJ707e3BAkEA1aIHbmcVfCP8\nY/uWuK0lvWxrIWfR5MlHhI8tD9lvkot2kyXiV+jB6/gktwk1QaFsy7dCXn7w03+k\nxrjEGGN+kQJBAMuKf55lDtU9K2Js3YSStTZAXP+Hz7XpoLxmbWFyGvBx806WjgAD\n624irwS+0tBxkERbRcisfb2cXmAx8earT9MCQDZuVCpjBWxd1t66qYpgQ29iAmG+\njBIY3qn9uOOC6RSTiCCx1FvFqDMxRFmGdRVFxeyZwsVE3qNksF0Zko0MPKECQCEe\noDV97DP2iCCz5je0R5hUUM2jo8DOC0GcyR+aGZgWcqjPBrwp5x08t43mHxeb4wW8\ndFZ6+trnntO4TMxkA9ECQB+yCPgO1zisJWYuD46KISoesYhwHe5C1BQElQgi9bio\nU39fFo88w1pok23a2CZBEXguSvCvexeB68OggdDXvy0=\n-----END RSA PRIVATE KEY-----\n";

// String to encrypt
char *str = "1234";

ERR_load_crypto_strings();  

BIO *bpo = BIO_new_mem_buf(b64_pKey, -1);
RSA *pubKey = PEM_read_bio_RSA_PUBKEY(bpo, NULL, NULL, NULL);

if ( !pubKey ) {
    printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
    return;
}

int rsa_length = RSA_size(pubKey);

BIO *b64 = NULL;
BIO *bmem = NULL;
BUF_MEM *bptr = NULL;

unsigned char encrypted[2560] = { 0 };
unsigned char retencrypted[2560] = { 0 };

int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING);
if ( resultEncrypt == -1 ) {
    printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
    return;
}

/*
 * Show base 64 encrypted string  
 */ 
b64 = BIO_new((BIO_METHOD *)BIO_f_base64());
BIO_set_flags(b64,BIO_FLAGS_BASE64_NO_NL);
bmem = BIO_new(BIO_s_mem());
b64 = BIO_push(b64, bmem);
BIO_write(b64, encrypted, resultEncrypt);
BIO_flush(b64);
BIO_get_mem_ptr(b64, &bptr);

memcpy(retencrypted, bptr->data, bptr->length);
BIO_free(b64);
BIO_free(bpo);
RSA_free(pubKey);

printf("Encrypted string:%s\n",retencrypted);

/*
 * Now decrypt this very string with the private key
 */ 

BIO *bpop = BIO_new_mem_buf(b64priv_key, -1);
RSA *privKey = PEM_read_bio_RSAPrivateKey(bpop, NULL, NULL, NULL);

if ( !privKey ) {
    printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
    return;
}

rsa_length = RSA_size(privKey);

unsigned char decrypted[2560] = { 0 };

int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING);

if ( resultDecrypt == -1 ) {
    printf("%s\n", ERR_error_string(ERR_get_error(), NULL));
    return;
}

printf("resultDecrypt=%d\ndecrypted string: %s\n",resultDecrypt,decrypted);
BIO_free(bpop);
RSA_free(privKey);
ERR_free_strings();
}

Note: I exported the private key using

openssl rsa -in rsa_privatekey.pem -check

and the public key:

openssl rsa -in rsa_privatekey.pem -pubout

Why am I getting the error?

Answer 1

The problem is that you're trying to decrypt the base64 encoded result. You should try to decrypt the result of the encryption.

That is, instead of:

int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), retencrypted, decrypted, privKey, PADDING);

You should call:

int resultDecrypt = RSA_private_decrypt( RSA_size(privKey), encrypted, decrypted, privKey, PADDING);

Also, there is a problem in the encryption call:

int resultEncrypt = RSA_public_encrypt(PADDING, str, encrypted, pubKey, PADDING);

Why are you passing PADDING as flen? This should be the length of the string (i.e. 4 or 5, depending on whether you want to encrypt the null character).

If you want to write the encrypted string as ASCII (encoded using base64), that's fine. But you have to decode it back before you decrypt it.