Implement PBEWITHHMACSHA512ANDAES_256 of java jasypt in python

Question

I am trying to encrypt a password in python and decrypt it in java springboot application using the jasypt library through jasypt plugin.

What i have done so far

• For simplicity i have used a zero salt and a fixed iv
• I have written the python script to perform the encryption using hselvarajan's pkcs12kdf

  import sys
  import math
  import base64
  import hashlib
  from Crypto.Cipher import AES
  from Crypto.Hash import SHA512
  
  from binascii import hexlify
  from binascii import unhexlify
  
  PY2 = sys.version_info[0] == 2
  PY3 = sys.version_info[0] == 3
  if PY2:
          str_encode = lambda s: str(s)
  elif PY3:
          str_encode = lambda s: str(s, 'utf-8')
  
  iterations          = 10000
  salt_block_size     = AES.block_size
  key_size            = 256
  
  password             = "test1"
  plaintext_to_encrypt = "password1"
  salt                 = "0000000000000000"
  iv                   = "0000000000000000"
  
  # -----------------------------------------------------------------------------
  # This is a pure copy paste of
  #  https://github.com/hselvarajan/pkcs12kdf/blob/master/pkcs12kdf.py
  # -----------------------------------------------------------------------------
  class PKCS12KDF:
          """This class generates keys and initialization vectors from passwords as specified in RFC 7292"""
  
          #
          # IDs for Key and IV material as in RFC
          #
  
          KEY_MATERIAL = 1
          IV_MATERIAL = 2
  
          def __init__(self, password, salt, iteration_count, hash_algorithm, key_length_bits):
                  self._password = password
                  self._salt = salt
                  self._iteration_count = iteration_count
                  self._block_size_bits = None
                  self._hash_length_bits = None
                  self._key_length_bytes = key_length_bits/8
                  self._key = None
                  self._iv = None
                  self._hash_algorithm = hash_algorithm
          #
          # Turns a byte array into a long
          #
  
          @staticmethod
          def byte_array_to_long(byte_array, nbytes=None):
                  #
                  # If nbytes is not present
                  #
                  if nbytes is None:
                          #
                          # Convert byte -> hex -> int/long
                          #
                          return int(hexlify(byte_array), 16)
                  else:
                          #
                          # Convert byte -> hex -> int/long
                          #
                          return int(hexlify(byte_array[-nbytes:]), 16)
  
          #
          # Turn a long into a byte array
          #
  
          @staticmethod
          def long_to_byte_array(val, nbytes=None):
                  hexval = hex(val)[2:-1] if type(val) is long else hex(val)[2:]
                  if nbytes is None:
                          return unhexlify('0' * (len(hexval) & 1) + hexval)
                  else:
                          return unhexlify('0' * (nbytes * 2 - len(hexval)) + hexval[-nbytes * 2:])
  
          #
          # Run the PKCS12 algorithm for either the key or the IV, specified by id
          #
  
          def generate_derived_parameters(self, id):
  
                  #
                  # Let r be the iteration count
                  #
  
                  r = self._iteration_count
  
                  if self._hash_algorithm not in hashlib.algorithms_available:
                          raise NotImplementedError("Hash function: "+self._hash_algorithm+" not available")
  
                  hash_function = hashlib.new(self._hash_algorithm)
  
                  #
                  # Block size, bytes
                  #
                  #v = self._block_size_bits / 8
                  v = hash_function.block_size
  
                  #
                  # Hash function output length, bits
                  #
                  #u = self._hash_length_bits / 8
                  u = hash_function.digest_size
  
                  # In this specification however, all passwords are created from BMPStrings with a NULL
                  # terminator. This means that each character in the original BMPString is encoded in 2
                  # bytes in big-endian format (most-significant byte first). There are no Unicode byte order
                  # marks. The 2 bytes produced from the last character in the BMPString are followed by
                  # two additional bytes with the value 0x00.
  
                  password = (unicode(self._password) + u'\0').encode('utf-16-be') if self._password is not None else b''
  
                  #
                  # Length of password string, p
                  #
                  p = len(password)
  
                  #
                  # Length of salt, s
                  #
                  s = len(self._salt)
  
                  #
                  # Step 1: Construct a string, D (the "diversifier"), by concatenating v copies of ID.
                  #
  
                  D = chr(id) * v
  
                  #
                  # Step 2: Concatenate copies of the salt, s, together to create a string S of length v * [s/v] bits (the
                  # final copy of the salt may be truncated to create S). Note that if the salt is the empty
                  # string, then so is S
                  #
  
                  S = b''
  
                  if self._salt is not None:
                          limit = int(float(v) * math.ceil((float(s)/float(v))))
                          for i in range(0, limit):
                                  S += (self._salt[i % s])
                  else:
                          S += '0'
  
                  #
                  # Step 3: Concatenate copies of the password, p, together to create a string P of length v * [p/v] bits
                  # (the final copy of the password may be truncated to create P). Note that if the
                  # password is the empty string, then so is P.
                  #
  
                  P = b''
  
                  if password is not None:
                          limit = int(float(v) * math.ceil((float(p)/float(v))))
                          for i in range(0, limit):
                                  P += password[i % p]
                  else:
                          P += '0'
  
                  #
                  # Step 4: Set I=S||P to be the concatenation of S and P.\00\00
                  #
  
                  I = bytearray(S) + bytearray(P)
  
                  #
                  # 5. Set c=[n/u]. (n = length of key/IV required)
                  #
  
                  n = self._key_length_bytes
                  c = int(math.ceil(float(n)/float(u)))
  
                  #
                  # Step 6 For i=1, 2,..., c, do the following:
                  #
  
                  Ai = bytearray()
  
                  for i in range(0, c):
                          #
                          # Step 6a.Set Ai=Hr(D||I). (i.e. the rth hash of D||I, H(H(H(...H(D||I))))
                          #
  
                          hash_function = hashlib.new(self._hash_algorithm)
                          hash_function.update(bytearray(D))
                          hash_function.update(bytearray(I))
  
                          Ai = hash_function.digest()
  
                          for j in range(1, r):
                                  hash_function = hashlib.sha256()
                                  hash_function.update(Ai)
                                  Ai = hash_function.digest()
  
                          #
                          # Step 6b: Concatenate copies of Ai to create a string B of length v bits (the final copy of Ai
                          # may be truncated to create B).
                          #
  
                          B = b''
  
                          for j in range(0, v):
                                  B += Ai[j % len(Ai)]
  
                          #
                          # Step 6c: Treating I as a concatenation I0, I1,..., Ik-1 of v-bit blocks, where k=[s/v]+[p/v],
                          # modify I by setting Ij=(Ij+B+1) mod 2v for each j.
                          #
  
                          k = int(math.ceil(float(s)/float(v)) + math.ceil((float(p)/float(v))))
  
                          for j in range(0, k-1):
                                  I = ''.join([
                                          self.long_to_byte_array(
                                                  self.byte_array_to_long(I[j:j + v]) + self.byte_array_to_long(bytearray(B)), v
                                          )
                                  ])
  
                  return Ai[:self._key_length_bytes]
  
          #
          # Generate the key and IV
          #
  
          def generate_key_and_iv(self):
                  self._key = self.generate_derived_parameters(self.KEY_MATERIAL)
                  self._iv = self.generate_derived_parameters(self.IV_MATERIAL)
                  return self._key, self._iv
  
  # -----------------------------------------------------------------------------
  # Main execution
  # -----------------------------------------------------------------------------
  kdf = PKCS12KDF(
      password        = password,
      salt            = salt,
      iteration_count = iterations,
      hash_algorithm  = "sha512",
      key_length_bits = key_size
  )
  (key, iv_tmp) = kdf.generate_key_and_iv()
  aes_key = key[:32]
  
  pad = salt_block_size - len(plaintext_to_encrypt) % salt_block_size
  plaintext_to_encrypt = plaintext_to_encrypt + pad * chr(pad)
  
  cipher = AES.new(aes_key, AES.MODE_CBC, iv)
  encrypted = cipher.encrypt(plaintext_to_encrypt)
  
  # Since we selt the salt to be zero's,
  # jasypt needs only the iv + encrypted value,
  # not the salt + iv + encrypted
  result = str_encode(base64.b64encode(iv + encrypted))
  
  # Python output : MDAwMDAwMDAwMDAwMDAwMKWsWH+Ku37n7ddfj0ayxp8=
  # Java output   : MDAwMDAwMDAwMDAwMDAwMAtqAfBtuxf+F5qqzC8QiFc=
  print(result)
  
  

  Run it as

  python2.7 test-PBEWITHHMACSHA512ANDAES_256.py
  paxYf4q7fuft11+PRrLGnw==
  

• I have written a unit test in jasypt repository to decrypt See PBEWITHHMACSHA512ANDAES_256EncryptorTest. Run it as

  $ cd jasypt
  $ mvn clean test -Dtest=org.jasypt.encryption.pbe.PBEWITHHMACSHA512ANDAES_256EncryptorTest
  

The problem: The above setup produces different results in python and in java

• Python output : MDAwMDAwMDAwMDAwMDAwMKWsWH+Ku37n7ddfj0ayxp8=
• Java output : MDAwMDAwMDAwMDAwMDAwMAtqAfBtuxf+F5qqzC8QiFc=

What i know

• The failure is due to not using the using the correct key in python. Adding additional logs, the error is

  EncryptionOperationNotPossibleException: javax.crypto.BadPaddingException: Given final block not properly padded. Such issues can arise if a bad key is used during decryption.
  

• The PBEWITHHMACSHA512ANDAES_256 uses the pkcs12 key derivation function. I do not understand where the HMAC is being used.
• I have also tried using the folling implementation to no avail. I am getting "" error in all of them.
  • oscrypto
  • python-hkdf
  • Cryptodome.Protocol.KDF HKDF I do not understand where the iterations are being used here.

  self.aes_key = HKDF(master = self.password, key_len = 32, salt = self.salt, hashmod = SHA512, num_keys = 1)
  

I would like some guidance on what i am doing wrong. Any help, any pointers would be much appreciated.

---

Update following Cryptodome's PBKDF2 and AES Here is the python script

import sys
import base64
from Cryptodome.Cipher import AES
from Cryptodome.Hash import SHA512
from Cryptodome.Protocol.KDF import PBKDF2
from Cryptodome.Util.Padding import pad

iterations          = 10000
password             = b'test1'
plaintext_to_encrypt = b'password1'
salt                 = b'0000000000000000'
iv                   = b'0000000000000000'

# -----------------------------------------------------------------------------
# Main execution
# -----------------------------------------------------------------------------
keys = PBKDF2(password, salt, 64, count=iterations, hmac_hash_module=SHA512)
aes_key = keys[:32]

cipher = AES.new(aes_key, AES.MODE_CBC, iv)
ct_bytes = cipher.encrypt(pad(plaintext_to_encrypt, AES.block_size))
encrypted = base64.b64encode(ct_bytes).decode('utf-8')

# Since we selt the salt to be zero's,
# jasypt needs only the iv + encrypted value,
# not the salt + iv + encrypted
result = encrypted

# Python output : 6tCAZbswCh9DZ1EK8utRuA==
# Java output   : C2oB8G27F/4XmqrMLxCIVw==
print(result)

and its output

python2.7 test-PBEWITHHMACSHA512ANDAES_256-2.py
6tCAZbswCh9DZ1EK8utRuA==

I try to decrypt it in java with the following error using the test

mvn clean test -Dtest=org.jasypt.encryption.pbe.PBEWITHHMACSHA512ANDAES_256EncryptorTest

[...]

Running org.jasypt.encryption.pbe.PBEWITHHMACSHA512ANDAES_256EncryptorTest
Test encr: C2oB8G27F/4XmqrMLxCIVw==
Error: javax.crypto.BadPaddingException: Given final block not properly padded. Such issues can arise if a bad key is used during decryption.Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 0.524 sec <<< FAILURE!
test1(org.jasypt.encryption.pbe.PBEWITHHMACSHA512ANDAES_256EncryptorTest)  Time elapsed: 0.522 sec  <<< ERROR!
org.jasypt.exceptions.EncryptionOperationNotPossibleException
    at org.jasypt.encryption.pbe.StandardPBEByteEncryptor.decrypt(StandardPBEByteEncryptor.java:1173)
    at org.jasypt.encryption.pbe.StandardPBEStringEncryptor.decrypt(StandardPBEStringEncryptor.java:738)
    at org.jasypt.encryption.pbe.PBEWITHHMACSHA512ANDAES_256EncryptorTest.test1(PBEWITHHMACSHA512ANDAES_256EncryptorTest.java:27)


Results :

Tests in error:
  test1(org.jasypt.encryption.pbe.PBEWITHHMACSHA512ANDAES_256EncryptorTest)

Tests run: 1, Failures: 0, Errors: 1, Skipped: 0

[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time:  8.648 s
[INFO] Finished at: 2020-06-24T17:40:04+08:00
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:2.12.4:test (default-test) on project jasypt: There are test failures.
[ERROR]
[ERROR] Please refer to /space/openbet/git/github-jasypt-jasypt/jasypt/target/surefire-reports for the individual test results.
[ERROR] -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException

Answer 1

PBEWITHHMACSHA512ANDAES_256 applies PBKDF2 to generate the key. Encryption is performed with AES-256, CBC.

The (originally) posted Jasypt test function used RandomIvGenerator, which creates a random IV. For the salt, ZeroSaltGenerator is applied, which generates a salt consisting of 16 zero bytes.

To implement the Python function you are looking for, it is best to use a fixed IV, e.g. with StringFixedIvGenerator. StringFixedSaltGenerator provides a corresponding functionality for the salt (FixedStringSaltGenerator has the same functionality but is deprecated since 1.9.2). StringFixedSaltGenerator and StringFixedIvGenerator encode the passed string with UTF-8 by default (but another encoding can be specified), so that the salt (or IV) 0000000000000000 is hex encoded 0x30303030303030303030303030303030.

Note that a fixed salt and IV may only be used for testing. In practice, a new random salt and a new random IV must be used for each encryption. Since salt and IV are not secret, they are usually concatenated with the ciphertext on byte level (e.g. in the order salt, iv, ciphertext) and sent to the receiver, who separates the parts and uses them for decryption.

If the same parameters (especially the same salt and IV) are used on both sides, then encryption with Python and decryption with Java works.

Encryption with Python (PyCryptodome):

import base64
from Cryptodome.Cipher import AES
from Cryptodome.Hash import SHA512
from Cryptodome.Protocol.KDF import PBKDF2
from Cryptodome.Util.Padding import pad

# Key generation (PBKDF2)
iterations           = 10000
password             = b'test1'
plaintext_to_encrypt = b'password1'
salt                 = b'5432109876543210'
iv                   = b'0123456789012345'
key = PBKDF2(password, salt, 32, count=iterations, hmac_hash_module=SHA512)

# Encryption (AES-256, CBC)
cipher = AES.new(key, AES.MODE_CBC, iv)
ct_bytes = cipher.encrypt(pad(plaintext_to_encrypt, AES.block_size))
encrypted = base64.b64encode(ct_bytes).decode('utf-8')

print(encrypted) # Output: kzLd5qPlCLnHq5sT7LOXzQ==

Decryption with Java (Jasypt):

StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor();
encryptor.setPassword("test1");
encryptor.setSaltGenerator(new StringFixedSaltGenerator("5432109876543210"));
encryptor.setIvGenerator(new StringFixedIvGenerator("0123456789012345"));
encryptor.setKeyObtentionIterations(10000);
encryptor.setAlgorithm("PBEWITHHMACSHA512ANDAES_256");
    
String decryptedMsg = encryptor.decrypt("kzLd5qPlCLnHq5sT7LOXzQ==");
System.out.println("Test decr: " + decryptedMsg); // Output: Test decr: password1

Answer 2

By the way, if anyone is still looking for this answer but with random salt and IV, it seems like they are appended to the cyphertext in order. Here is the encryption/decryption solution that is compatible with PBEWithHMACSHA512AndAES_256:

from base64 import b64decode, b64encode
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.padding import PKCS7
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes

KEY = b'my awesome key'

def decrypt_pbe_with_hmac_sha512_aes_256(obj: str) -> str:
    # re-generate key from
    encrypted_obj = b64decode(obj)
    salt = encrypted_obj[0:16]
    iv = encrypted_obj[16:32]
    cypher_text = encrypted_obj[32:]
    kdf = PBKDF2HMAC(hashes.SHA512(), 32, salt, 1000, backend=default_backend())
    key = kdf.derive(KEY)

    # decrypt
    cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
    decryptor = cipher.decryptor()
    padded_text = decryptor.update(cypher_text) + decryptor.finalize()

    # remove padding
    unpadder = PKCS7(128).unpadder()
    clear_text = unpadder.update(padded_text) + unpadder.finalize()
    return clear_text.decode()


def encrypt_pbe_with_hmac_sha512_aes_256(obj: str, salt: bytes = None, iv: bytes = None) -> str:
    # generate key
    salt = salt or os.urandom(16)
    iv = iv or os.urandom(16)
    kdf = PBKDF2HMAC(hashes.SHA512(), 32, salt, 1000, backend=default_backend())
    key = kdf.derive(KEY)

    # pad data
    padder = PKCS7(128).padder()
    data = padder.update(obj.encode()) + padder.finalize()

    # encrypt
    cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
    encryptor = cipher.encryptor()
    cypher_text = encryptor.update(data) + encryptor.finalize()

    return b64encode(salt + iv + cypher_text).decode()

Then you can use it directly using the base64 output of Jasypt:

>>> decrypt_pbe_with_hmac_sha512_aes_256(encrypt_pbe_with_hmac_sha512_aes_256('hello world'))
'hello world'