Decrypting EnvelopedCms with non-default AlgorithmIdentifier

Question

I'm trying to decrypt a EnvelopedCms that was encrypted using a non-default AlgorithmIdentifier like this:

ContentInfo contentInfo = new ContentInfo(data);
EnvelopedCms envelopedCms = new EnvelopedCms(contentInfo, new AlgorithmIdentifier(new System.Security.Cryptography.Oid("2.16.840.1.101.3.4.1.42")));
CmsRecipientCollection recipients = new CmsRecipientCollection(SubjectIdentifierType.IssuerAndSerialNumber, certificates);
envelopedCms.Encrypt(recipients);
byte[] encryptedData = envelopedCms.Encode();

The encryption works as expected. Now when I try to decrypt the envelopedCms using something like this:

EnvelopedCms envelopedCms = new EnvelopedCms();
envelopedCms.Decode(encryptedData );
envelopedCms.Decrypt(certificates);
byte[] decryptedData = envelopedCms.ContentInfo.Content;

I notice that a.) the access to the certificate takes quite long (longer then when using the default AlgorithmIdentifier) and b.) I get this error message:

System.Security.Cryptography.CryptographicException: Access was denied because of a security violation.

Which, looking at the source where this fails, is probably not the issue. Can anyone get the decrypt code above working [with a smartcard]?

//EDIT1 Please note that this issue only occures if the certificate used is placed on a smartcard AND if a AlgorithmIdentifier other then the default one (3DES) was specified, as in the example code. Everything works fine if either the default AlgorithmIdentifier is used or the certificate is NOT placed on a smartcard. It doesn't seem like a SC issue per se, since it's working with the default AlgorithmIdentifier. It's rather the combination of a SC and the AES AlgorithmIdentifier used that's causing the issue but I was unable to find a working solution.

//EDIT2 A complete example demonstrating the issue, read comments for details:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
using System.Security.Cryptography;
using System.IO;
using System.Reflection;
using System.Diagnostics;
using System.Runtime.Serialization;
using System.Security.Cryptography.Pkcs;

namespace ConsoleApp
{

    class Program
    {
        static void Main(string[] args)
        {
            // Select the (smartcard) certificate to use it for encryption
            X509Store store = new X509Store(StoreName.My, StoreLocation.CurrentUser);
            store.Open(OpenFlags.ReadOnly | OpenFlags.OpenExistingOnly);
            X509Certificate2Collection collection = (X509Certificate2Collection)store.Certificates;
            X509Certificate2Collection fcollection = (X509Certificate2Collection)collection.Find(X509FindType.FindByTimeValid, DateTime.Now, false);
            X509Certificate2Collection scollection = X509Certificate2UI.SelectFromCollection(fcollection, "Certificate Select", "Select your smartcard certificate", X509SelectionFlag.MultiSelection);

            // Output which certificate will be used
            Console.WriteLine("Using Certificate:");
            int i = 0;
            foreach (X509Certificate2 x509 in scollection)
            {
                byte[] rawdata = x509.RawData;
                Console.WriteLine("---------------------------------------------------------------------");
                Console.WriteLine("1.\tFull DN: {0}", x509.Subject);
                Console.WriteLine("\tThumbprint: {0}", x509.Thumbprint);
                Console.WriteLine("---------------------------------------------------------------------");
                i++;
            }
            store.Close();

            // Wait
            Console.WriteLine("Press any key to continue...");
            Console.ReadKey(true);

            // Create data for encryption
            string message = "THIS IS OUR SECRET MESSAGE";
            byte[] data = System.Text.Encoding.ASCII.GetBytes(message);

            // Encrypt
            Console.WriteLine("Encrypting message...");

            // ContentInfo contentInfo = new ContentInfo(data); // will use default ContentInfo Oid, which is "DATA"
            // Explicitly use ContentInfo Oid 1.2.840.113549.1.7.1, "DATA", which is the default.
            ContentInfo contentInfo = new ContentInfo(new System.Security.Cryptography.Oid("1.2.840.113549.1.7.1"), data);

            // If using OID 1.2.840.113549.3.7 (the default one used if empty constructor is used) or 1.2.840.113549.1.9.16.3.6  everything works
            // If using OID 2.16.840.1.101.3.4.1.42 (AES CBC) it breaks
            AlgorithmIdentifier encryptionAlgorithm = new AlgorithmIdentifier(new System.Security.Cryptography.Oid("1.2.840.113549.3.7"));
            // EnvelopedCms envelopedCms = new EnvelopedCms(contentInfo); // this will use default encryption algorithm (3DES)
            EnvelopedCms envelopedCms = new EnvelopedCms(contentInfo, encryptionAlgorithm);
            Console.WriteLine("Encyption Algorithm:" + envelopedCms.ContentEncryptionAlgorithm.Oid.FriendlyName);
            Console.WriteLine("Encyption Algorithm:" + envelopedCms.ContentEncryptionAlgorithm.Oid.Value);
            CmsRecipientCollection recipients = new CmsRecipientCollection(SubjectIdentifierType.IssuerAndSerialNumber, scollection);
            /*Console.WriteLine("Receipientinfo count: " + encryptionEnvelopedCms.RecipientInfos.Count.ToString());
            foreach (var i in encryptionEnvelopedCms.RecipientInfos)
            {
                Console.Write("RecipientInfo Encryption Oid: " + i.KeyEncryptionAlgorithm.Oid);
            }
            */
            envelopedCms.Encrypt(recipients);
            byte[] encryptedData = envelopedCms.Encode();
            Console.WriteLine("Message encrypted!");

            // Decrypt
            envelopedCms.Decode(encryptedData);
            Console.WriteLine("Decryption Algorithm:" + envelopedCms.ContentEncryptionAlgorithm.Oid.FriendlyName);
            Console.WriteLine("Decryption Algorithm:" + envelopedCms.ContentEncryptionAlgorithm.Oid.Value);
            // Next line will fail if both conditions are true: 
            // 1. A non-default AlgorithmIdentifier was used for encryption, in our case AES
            // 2. The private key required for decryption is placed on a smartcard that requires a manual action, such as entering a PIN code, before releasing the private key
            // Note that everything works just fine when the default AlgorithmIdentifier is used (3DES) or the private key is available in the X509Store
            envelopedCms.Decrypt(scollection);
            byte[] decryptedData = envelopedCms.ContentInfo.Content;
            Console.WriteLine("Message decrypted!");
            Console.WriteLine("Decrypted message: " + System.Text.Encoding.ASCII.GetString(decryptedData));
            Console.WriteLine("Press any key to exit.");
            Console.ReadKey(true);
        }
    }
}