OpenSSL and MS CryptoAPI compatibility issue

I'm facing a incompatibility issue between Microsoft CrypoAPI and OpenSSL and I can't solve it.

I want to RSA encrypt a message using CAPI with public key and after that to decrypt it with OpenSSL (using the private key), but with no luck.

The steps I'm doing are as follow:

Generating a public/private key pair with OpenSSL in pem format.

openssl genrsa -out private.pem 2048

openssl rsa -in private.pem -outform PEM -pubout -out public.pem

Then In CAPI, I'm importing the generated keys like this: dsadsa

      const char* szPemPrivKey = 
"-----BEGIN RSA PRIVATE KEY-----"
"MIIEpAIBAAKCAQEAvUT7PaQzzhSmyQKrnroWCk2tdy9O1BR6bINObhGXoDfK4tnU"
"qgpr0MbM8VjCvpgNjJT8m5RZkvSxcmaWZZQnXxr/SESnrmwk6CcRAvEk45M3LTix"
"N0TTUZzQdBbG9z0bgx1a1P866S87MOf6wbb9yUfV79N7J+xZRQC8mnRtvmzjRZ3W"
"MOcBZ8n1C106gOq//SwvB28mGWTpJ+opASv3mkxTydiDd5v7/yq2Lx4NrvJN+1E4"
"nOX6PLiTECFcvNYzZe9KDuJ5CJv45ifRUo9m5ebo759lKpL1X69ptG7RpHEYULYY"
"urwEGExC1jp/Nbft0zOehsF4wwFqwQVQ34m5LQIDAQABAoIBAD6dQISu+th1aovb"
"T01ugHYeJoHka66rq6iUc/Dj7wZ5DqynpbwvQGXMLua1F5AYG3tjmoIZvNxqCP4w"
"xBaMwc2rz8JnvBVu/3Kx4eXYQvzqqflS5QXExigcubV+B4qpc52Xq4IFgca88lcG"
"l6VYVXMuSa9Shk652PqD+OEcHWY+aygXETLGEAlO88iWe2LbzrD9I1faW8Mrj+wi"
"b9mVhZbjbrI+w9O7cfde4d9Lo4wzGrrunRa0THKUyfwJGtTJ3eP2jCWlE1ij3Xmy"
"c/Mf8U8RlMU8N/Ys77WtCXUA3DK0ge8HSITqvD1NHSyuPM0XqinTkRSg0Ri/tWo+"
"1C6gKJ0CgYEA9K9snRlfuQshwfyR40npJ8r5X9YApjfK8PrxsmdpOi8tpDyi5I66"
"fmnUsbV0ikM+9U898myUAF3SxZIZfw+LYI3ofdstnv6wc9+c9jP9wbGxSryiBurW"
"d3uPuemee7+CxTVEwI0PEAVerRSP+m6MZ/F+SRkQjyLmBXu7soxaspMCgYEAxgWL"
"/wkz6WCf37+9TUHO0MG7vOQ2sI7Bc+82dAa9fsxLwpncmkNxEv+hK3k7Jlr2J4pZ"
"SveRhcq3Ohm2aDRi5CfgQaZO4bGDop7ZYWZuW/MakZdCf8olWCifXEPWcxBo8FGW"
"9/3XEbXkW481HDrX4wyn2b3ptdSqMcdDMKmifT8CgYEA6vtVWYG2teSE5OED0b13"
"VinNV0YTlY1bLhYw6134ZlJMiL9ayBhx7VkBVDCo3Oc7nSYenaO8dqWj9u0Z4zYw"
"aeeecM9+foSlPIJxINhJSCy30Mha6j24/UICg05iTwFaOr2vayOMZZxikeF/a8ei"
"u2fmGZkil/Ox524ukYfMylUCgYAiFTJTYzIcKRVbXZUnhvwh0jaN/HmtSeTiH3ov"
"3jkfaepgRDtEEfeUXYtQAD2+DEnx5E4aKSJS9OE0jthmdx3OR07B/e31yqfthYnE"
"yeyUxvL7vB0mAZUL53IGX2a5x0lIk8J4TKiH09bKK4von/gojDUXtShUs5XXm4Rl"
"C8174QKBgQDbRGyg3VCwo3p0sdqm5UlCL0pc36T5bBUfkVGpJdBZxbIx283CvY35"
"OGqcYdiANMn+alg9IDXfuaFYgg1QTQSkF74CMi+gY7Z8n2OKsjF0cR8VkIYoI9BL"
"iDXvlWs2QVsv+1CIYmZdI8nkExirzRvBD75ZqNdGSWkrfuYnr/bpHg=="
"-----END RSA PRIVATE KEY-----";


const char* szPemPubKey = 
"-----BEGIN PUBLIC KEY-----"
"MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvUT7PaQzzhSmyQKrnroW"
"Ck2tdy9O1BR6bINObhGXoDfK4tnUqgpr0MbM8VjCvpgNjJT8m5RZkvSxcmaWZZQn"
"Xxr/SESnrmwk6CcRAvEk45M3LTixN0TTUZzQdBbG9z0bgx1a1P866S87MOf6wbb9"
"yUfV79N7J+xZRQC8mnRtvmzjRZ3WMOcBZ8n1C106gOq//SwvB28mGWTpJ+opASv3"
"mkxTydiDd5v7/yq2Lx4NrvJN+1E4nOX6PLiTECFcvNYzZe9KDuJ5CJv45ifRUo9m"
"5ebo759lKpL1X69ptG7RpHEYULYYurwEGExC1jp/Nbft0zOehsF4wwFqwQVQ34m5"
"LQIDAQAB"
"-----END PUBLIC KEY-----";



/* 
..... some unrelated code here */
char           derPubKey[2048];
size_t         derPubKeyLen = 2048;
CERT_PUBLIC_KEY_INFO *publicKeyInfo;
int            publicKeyInfoLen;
HCRYPTPROV hProv = NULL;
HCRYPTKEY hKey = NULL;
/*
* Convert from PEM format to DER format - removes header and footer and decodes from base64
*/
if ( !CryptStringToBinaryA( szPemPubKey, 0, CRYPT_STRING_BASE64HEADER, (BYTE*)derPubKey, (DWORD*)&derPubKeyLen, NULL, NULL ) )
{
    fprintf( stderr, "CryptStringToBinary failed. Err: %d\n", GetLastError() );
}

/*
 * Decode from DER format to CERT_PUBLIC_KEY_INFO
 */
if ( !CryptDecodeObjectEx( X509_ASN_ENCODING, X509_PUBLIC_KEY_INFO, (BYTE*)derPubKey, derPubKeyLen, 
                           CRYPT_ENCODE_ALLOC_FLAG, NULL, &publicKeyInfo, (DWORD*)&publicKeyInfoLen ) )
{
    fprintf( stderr, "CryptDecodeObjectEx 1 failed. Err: %p\n", GetLastError() );
    return -1;
}

// Create a temporary and volatile CSP context in order to import
// the key and use for signing
if (!CryptAcquireContext(&hProv, NULL, MS_ENHANCED_PROV, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
{
    printf("CryptAcquireContext failed with error 0x%.8X\n", GetLastError());
    goto main_exit;
}

/*
 * Import the public key using the context
 */
if ( !CryptImportPublicKeyInfo( hProv, X509_ASN_ENCODING, publicKeyInfo, &hKey ) )
{
    fprintf( stderr, "CryptImportPublicKeyInfo failed. error: %d\n", GetLastError() );
    return -1;
}

It seems that the key is imported correctly and hKey is populated.

So far, so good. Then I'm tring to encrypt a plaintext

    char* plaintext = "123456";
LPBYTE pEncryptedData = NULL;
DWORD EncryptedDataLen = 0;

LPBYTE pEncryptedDataRev = NULL; /* I'll explain this later (: */
DWORD EncryptedDataRevLen = 0;

pEncryptedData = (LPBYTE) LocalAlloc(0, 500);
EncryptedDataLen = 6;


pEncryptedDataRev = (LPBYTE) LocalAlloc(0, 500);
EncryptedDataLen = 6;


CopyMemory(pEncryptedData, plaintext, 6);


if ( CryptEncrypt( hKey, NULL, TRUE, 0, pEncryptedData, &EncryptedDataLen, 500) ) {
    DWORD dwBytesWritten = 0;

    hFile = CreateFile(L"poc_enc",              // name of the write
                       GENERIC_WRITE,          // open for writing
                       0,                      // do not share
                       NULL,                   // default security
                       CREATE_NEW,             // create new file only
                       FILE_ATTRIBUTE_NORMAL,  // normal file
                       NULL);                  // no attr. template

    ReverseStream(pEncryptedData, pEncryptedDataRev, EncryptedDataLen);

    WriteFile( 
                    hFile,           // open file handle
                    pEncryptedData,      // start of data to write
                    EncryptedDataLen,  // number of bytes to write
                    &dwBytesWritten, // number of bytes that were written
                    NULL);            // no overlapped structure

      CloseHandle(hFile); 
}

You can see that, I'm reversing the encrypted data with ReverseStrem(), that is because, that I've read that the endiannes which CAPI and OpenSSL use differ. (little-endian CAPI, and big-endian OpenSSL)

void ReverseStream(LPBYTE Source, LPBYTE Destination, DWORD Size)
{
    DWORD cnt = Size;
    while(0<cnt)
    {
        Destination[Size-cnt] = Source[cnt];
        cnt--;
    }
}

OK, Now the troubles come. The encrypted plaintext (123456) is written to a file named poc_enc, and when I try to decrypt it with OpenSSL

openssl rsautl -decrypt -in poc_enc -out plaintext -inkey private.pem

and I'm getting the following error:

    RSA operation error
16968:error:0407109F:rsa routines:RSA_padding_check_PKCS1_type_2:pkcs decoding error:.\crypto\rsa\rsa_pk1.c:273:
16968:error:04065072:rsa routines:RSA_EAY_PRIVATE_DECRYPT:padding check failed:.\crypto\rsa\rsa_eay.c:602:

Any ideas how can I fix this issue? Thanks :)

Solution

The reverse function is wrong, this way you miss the last byte, it should be something like this:

void ReverseStream(LPBYTE Source, LPBYTE Destination, DWORD Size)
{
    int SourceCnt = Size;
    int DestCnt = 0;

    for (SourceCnt = Size - 1, DestCnt = 0; SourceCnt >= 0; SourceCnt--, DestCnt++)
        Destination[DestCnt] = Source[SourceCnt];
}