Decryption failure using openssl EVP_CipherFinal_ex

Question

I am trying to encrypt and decrypt a buffer using openssl.

I took some example code from here (see the 2nd do_crypt function towards the end of the page) and modified it to take the message to encrypt/decrypt from a vector instead of an input file.

Unfortunately, during the decryption stage, the EVP_CipherFinal_ex funcition fails, and I don't know why.

I have tried to make a compilable example from my code (but i haven't been able to test it), that will show you what i've done so far :

#include <iostream>
#include <vector>
#include <openssl/evp.h>

using namespace std;
typedef vector<uint8_t> Vect;

Vect _cipherMessage(const Vect& ai_in, const Vect& ai_key, const Vect& ai_iv, int ai_encode_decode)
{
    cout << ((ai_encode_decode==1)?"ENCODE":"DECODE") << endl;
    Vect w_ret;
    uint8_t outbuf[1024 + EVP_MAX_BLOCK_LENGTH];
    int outlen;
    
    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
    EVP_CipherInit_ex(ctx, EVP_aes_128_cbc(), NULL, ai_key.data(), ai_iv.data(), ai_encode_decode);
    
    if (!EVP_CipherUpdate(ctx, outbuf, &outlen, ai_in.data(), ai_in.size()))
    {
        // Error
        cout << "EVP_CipherUpdate" << endl;
        EVP_CIPHER_CTX_free(ctx);
        return w_ret;
    }
    
    if (!EVP_CipherFinal_ex(ctx, outbuf + outlen, &outlen))
    {
        // Error
        cout << "EVP_CipherFinal_ex" << endl;
        EVP_CIPHER_CTX_free(ctx);
        return w_ret;
    }
    
    EVP_CIPHER_CTX_free(ctx);
    
    for (uint32_t i=0 ; i<outlen ; i++)
    {
        w_ret.push_back(outbuf[i]);
    }
    
    return w_ret;
}

int main()
{
    Vect w_inputFrame { 0x53, 0x69, 0x78, 0x74, 0x65, 0x65, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x6f, 0x6e, 0x67 }; // "SixteenBytesLong" in hex
    Vect w_key { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
    Vect w_iv { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };

    Vect w_encryptedFrame = _cipherMessage(w_inputFrame, w_key, w_iv, 1);
    Vect w_decryptedFrame = _cipherMessage(w_encryptedFrame, w_key, w_iv, 0);
    cout << "Encryption / Decryption : " << ((w_inputFrame == w_decryptedFrame)?"SUCCESS":"FAILURE") << endl;

    return 0;
}

Please could someone tell me what I am doing wrong ?

NOTE :

I have tested the openssl command on my device and was able to successfully encrypt / decrypt the message :

echo -n SixteenBytesLong | openssl enc -e -aes-128-cbc -K 0102030405060708090a0b0c0d0e0f -iv 0102030405060708090a0b0c0d0e0f -nosalt -nopad -base64 -p
key=0102030405060708090A0B0C0D0E0F00
iv =0102030405060708090A0B0C0D0E0F00
Ow74k7Pm+g8KQNiKZQptgw==

echo Ow74k7Pm+g8KQNiKZQptgw== | openssl enc -d -aes-128-cbc -K 0102030405060708090a0b0c0d0e0f -iv 0102030405060708090a0b0c0d0e0f -nosalt -nopad -base64 -p
key=0102030405060708090A0B0C0D0E0F00
iv =0102030405060708090A0B0C0D0E0F00
SixteenBytesLong

Answer 1

The main problem is that outlen is being overwritten the call to EVP_CipherFinal_ex. So your total outlen is incorrect when you get to the end of the funciton.

You could do something like:

Vect _cipherMessage(const Vect& ai_in, const Vect& ai_key, const Vect& ai_iv, int ai_encode_decode)
{
    cout << ((ai_encode_decode==1)?"ENCODE":"DECODE") << endl;
    Vect w_ret;
    uint8_t outbuf[1024 + EVP_MAX_BLOCK_LENGTH];
    int outlen;
    int totalOutlen;
    
    EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
    EVP_CipherInit_ex(ctx, EVP_aes_128_cbc(), NULL, ai_key.data(), ai_iv.data(), ai_encode_decode);
    
    if (!EVP_CipherUpdate(ctx, outbuf, &totalOutlen, ai_in.data(), (int)ai_in.size()))
    {
        // Error
        cout << "EVP_CipherUpdate: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
        EVP_CIPHER_CTX_free(ctx);
        return w_ret;
    }
    
    if (!EVP_CipherFinal_ex(ctx, outbuf + totalOutlen, &outlen))
    {
        // Error
        cout << "EVP_CipherFinal_ex : " << ERR_error_string(ERR_get_error(), nullptr) << endl;
        EVP_CIPHER_CTX_free(ctx);
        return w_ret;
    }
    totalOutlen += outlen;
    
    EVP_CIPHER_CTX_free(ctx);
    
    for (int i=0 ; i<totalOutlen ; i++)
    {
        w_ret.push_back(outbuf[i]);
    }
    
    return w_ret;
}

Another problem is that the if the ai_in is larger than (1024 + EVP_MAX_BLOCK_LENGTH) you are going overwrite and corrupt your stack.

You could do something like this:

namespace
{
    template<typename T, typename D>
    std::unique_ptr<T, D> make_handle(T* handle, D deleter)
    {
        return std::unique_ptr<T, D>{handle, deleter};
    }
}

Vect _cipherMessage(const Vect& ai_in, const Vect& ai_key, const Vect& ai_iv, int ai_encode_decode)
{
    cout << ((ai_encode_decode==1)?"ENCODE":"DECODE") << endl;
    Vect w_ret;
    int outlen;
    int totalOutlen;

    w_ret.resize(ai_in.size() + EVP_MAX_BLOCK_LENGTH);
    
    auto ctx = make_handle(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
    if(!ctx)
    {
        // Error
        cout << "EVP_CIPHER_CTX_new: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
        return w_ret;
    }

    if(!EVP_CipherInit_ex(ctx.get(), EVP_aes_128_cbc(), nullptr, ai_key.data(), ai_iv.data(), ai_encode_decode))
    {
        // Error
        cout << "EVP_CipherInit_ex: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
        return w_ret;
    }
    
    if (!EVP_CipherUpdate(ctx.get(), w_ret.data(), &totalOutlen, ai_in.data(), (int)ai_in.size()))
    {
        // Error
        cout << "EVP_CipherUpdate: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
        return w_ret;
    }
    
    if (!EVP_CipherFinal_ex(ctx.get(), w_ret.data() + totalOutlen, &outlen))
    {
        // Error
        cout << "EVP_CipherFinal_ex : " << ERR_error_string(ERR_get_error(), nullptr) << endl;
        return w_ret;
    }
    totalOutlen += outlen;
    w_ret.resize(totalOutlen);
    return w_ret;
}