How to resolve the "EVP_DecryptFInal_ex: bad decrypt" during file decryption

Question

I have the following query.Could any one please suggest me a solution.

I'm working on encryption and decryption of file for first time.

I have encrypted file through command prompt using the command:

openssl enc -aes-256-cbc -in file.txt -out file.enc -k "key value" -iv "iv value" 

I have to decrypt it programmatically. So I have written the program for it, but it is throwing the following error:

./exe_file enc_file_directory ... error: 06065064: digital envelope routines: EVP_DecryptFInal_ex: bad decrypt: evp_enc.c 

---

The program below takes input as directory path and search for encrypted file ".enc" and try to decrypt it read into buffer.

Code:

#include <stdio.h> #include <string.h> #include <stdlib.h> #include <dirent.h> #include <sys/stat.h> #include <sys/types.h> #include <openssl/evp.h> #include <openssl/err.h> #include <openssl/conf.h> #include <libxml/globals.h>  void handleErrors(char *msg) {     {         ERR_print_errors_fp(stderr);         printf("%s", msg);         abort();      } }  void freeMemory(char *mem) {     if (NULL != mem)     {         free(mem);         mem = NULL;     } }  /* Function to decrypt the XML files */  int decryptXML(unsigned char *indata, unsigned char *outdata, int fsize) {      int outlen1 = 0, outlen2 = 0;      unsigned char iv[] = "b63e541bc9ece19a1339df4f8720dcc3";     unsigned char ckey[] = "70bbc518c57acca2c2001694648c40ddaf19e3b4fe1376ad656de8887a0a5ec2" ;      if (NULL == indata)     {         printf ("input data is empty\n");         return 0;     }      if (0 >= fsize)     {         printf ("file size is zero\n");         return 0;     }      outdata = (char *) malloc (sizeof (char) * fsize * 2);      EVP_CIPHER_CTX ctx;      EVP_CIPHER_CTX_init(&ctx);      if (! EVP_DecryptInit_ex (&ctx, EVP_aes_256_cbc(), NULL, ckey, iv))     {         EVP_CIPHER_CTX_cleanup(&ctx);     handleErrors("DInit");     }      if (! EVP_DecryptUpdate (&ctx, outdata, &outlen1, indata, fsize))     {         EVP_CIPHER_CTX_cleanup(&ctx);         handleErrors("DUpdate");     }      if (! EVP_DecryptFinal_ex (&ctx, outdata + outlen1, &outlen2))     {          EVP_CIPHER_CTX_cleanup(&ctx);         handleErrors("DFinal");     }      EVP_CIPHER_CTX_cleanup(&ctx);      return outlen1+outlen2;  }  int isDirectory(char *path) {     DIR *dir = NULL;     FILE *fin = NULL, *fout = NULL;     int enc_len = 0, dec_len = 0, fsize = 0, ksize = 0;     unsigned char *indata = NULL, *outdata = NULL;     char buff[BUFFER_SIZE], file_path[BUFFER_SIZE], cur_dir[BUFFER_SIZE];      struct dirent *in_dir;     struct stat s;      if (NULL == (dir = opendir(path)))     {         printf ("ERROR: Failed to open the directory %s\n", path);         perror("cannot open.");         exit(1);     }      while (NULL != (in_dir = readdir(dir)))     {          if (!strcmp (in_dir->d_name, ".") || !strcmp(in_dir->d_name, ".."))             continue;          sprintf (buff, "%s/%s", path, in_dir->d_name);          if (-1 == stat(buff, &s))         {             perror("stat");             exit(1);         }          if (S_ISDIR(s.st_mode))         {              isDirectory(buff);         }         else         {             strcpy(file_path, buff);              if (strstr(file_path, ".enc"))             {                  /* File to be decrypted */                  fout = fopen(file_path,"rb");                   fseek (fout, 0L, SEEK_END);                 fsize = ftell(fout);                 fseek (fout, 0L, SEEK_SET);                  indata = (char*)malloc(fsize);                  fread (indata, sizeof(char), fsize, fout);                  if (NULL == fout)                 {                     perror("Cannot open enc file: ");                     return 1;                 }                   dec_len = decryptXML (indata, outdata, fsize);                 outdata[dec_len] = '\0';                 printf ("%s\n", outdata);                 fclose (fin);                 fclose (fout);              }         }     }        closedir(dir);     freeMemory(outdata);     freeMemory(indata);      return 1;  }   int main(int argc, char *argv[]) {     int result;      if (argc != 2)     {         printf ("Usage: <executable> path_of_the_files\n");         return -1;     }      ERR_load_crypto_strings();     OpenSSL_add_all_algorithms();     OPENSSL_config(NULL);      /* Checking for the directory existance */      result = isDirectory(argv[1]);      EVP_cleanup();     ERR_free_strings();      if (0 == result)         return 1;     else        return 0; } 

Thank you.

Answer 1

This message digital envelope routines: EVP_DecryptFInal_ex: bad decrypt can also occur when you encrypt and decrypt with an incompatible versions of openssl.

The issue I was having was that I was encrypting on Windows which had version 1.1.0 and then decrypting on a generic Linux system which had 1.0.2g.

It is not a very helpful error message!

---

Working solution:

A possible solution from @AndrewSavinykh that worked for many (see the comments):

Default digest has changed between those versions from md5 to sha256. One can specify the default digest on the command line as -md sha256 or -md md5 respectively

Answer 2

I experienced a similar error reply while using the openssl command line interface, while having the correct binary key (-K). The option "-nopad" resolved the issue:

Example generating the error:

echo -ne "\x32\xc8\xde\x5c\x68\x19\x7e\x53\xa5\x75\xe1\x76\x1d\x20\x16\xb2\x72\xd8\x40\x87\x25\xb3\x71\x21\x89\xf6\xca\x46\x9f\xd0\x0d\x08\x65\x49\x23\x30\x1f\xe0\x38\x48\x70\xdb\x3b\xa8\x56\xb5\x4a\xc6\x09\x9e\x6c\x31\xce\x60\xee\xa2\x58\x72\xf6\xb5\x74\xa8\x9d\x0c" | openssl aes-128-cbc -d -K 31323334353637383930313233343536 -iv 79169625096006022424242424242424 | od -t x1 

Result:

bad decrypt 140181876450560:error:06065064:digital envelope  routines:EVP_DecryptFinal_ex:bad decrypt:../crypto/evp/evp_enc.c:535: 0000000 2f 2f 07 02 54 0b 00 00 00 00 00 00 04 29 00 00 0000020 00 00 04 a9 ff 01 00 00 00 00 04 a9 ff 02 00 00 0000040 00 00 04 a9 ff 03 00 00 00 00 0d 79 0a 30 36 38 

Example with correct result:

echo -ne "\x32\xc8\xde\x5c\x68\x19\x7e\x53\xa5\x75\xe1\x76\x1d\x20\x16\xb2\x72\xd8\x40\x87\x25\xb3\x71\x21\x89\xf6\xca\x46\x9f\xd0\x0d\x08\x65\x49\x23\x30\x1f\xe0\x38\x48\x70\xdb\x3b\xa8\x56\xb5\x4a\xc6\x09\x9e\x6c\x31\xce\x60\xee\xa2\x58\x72\xf6\xb5\x74\xa8\x9d\x0c" | openssl aes-128-cbc -d -K 31323334353637383930313233343536 -iv 79169625096006022424242424242424 -nopad | od -t x1 

Result:

0000000 2f 2f 07 02 54 0b 00 00 00 00 00 00 04 29 00 00 0000020 00 00 04 a9 ff 01 00 00 00 00 04 a9 ff 02 00 00 0000040 00 00 04 a9 ff 03 00 00 00 00 0d 79 0a 30 36 38 0000060 30 30 30 34 31 33 31 2f 2f 2f 2f 2f 2f 2f 2f 2f 0000100