Search code examples
c++visual-studioencryptionopensslaes

OPENSSL ERROR : lib(6) func(101) reason(100) evp_enc.c


So I am working on AES encryption but this error keeps bugging me.

ERROR:

lib(6) func(101) reason(100) evp_enc.c

I am encrypting my file with one program and decrypting it with another. The encryption was successful with this code.

#include "stdafx.h"
#include <openssl/conf.h>
#include <openssl/evp.h>
#include <openssl/err.h>
#include <string.h>
#include <string>
#include <iostream>
#include <fstream>
#include <openssl/applink.c>

using namespace std;

void handleErrors(void)
{
    ERR_print_errors_fp(stderr);
    abort();
}
int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key,
    unsigned char *iv, unsigned char *ciphertext)
{
    EVP_CIPHER_CTX *ctx;

    int len;

    int ciphertext_len;

    /* Create and initialise the context */
    if (!(ctx = EVP_CIPHER_CTX_new())) handleErrors();


    if (1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, key, iv))
        handleErrors();

    if (1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len))
        handleErrors();
    ciphertext_len = len;


    if (1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) handleErrors();
    ciphertext_len += len;

    /* Clean up */
    EVP_CIPHER_CTX_free(ctx);
    // cout << ciphertext_len << "\n";
    return ciphertext_len;
}

using namespace std;
int main(void)
{
    /* A 256 bit key */
    unsigned char key[2000];
    memset(key, 0, sizeof(key));
    char s; int initializer = 0;
    /* Key reading */
    string path = "C:/openssl/mykey.pem";
    ifstream myfile(path);
    while (!myfile.eof())
    {
        myfile >> s;
        key[initializer] = s;
        initializer++;
        /* Key is read in such a way that each character is stored into the array */
    }

    cout << key;

    myfile.close();


    /* A 128 bit IV */
    unsigned char *iv = (unsigned char *)"0123456789012345";
    /* Message to be encrypted */
    //unsigned char text[]="weufhskgwesjfho";
    char text[2000];
    memset(text, 0, sizeof(text));
    // Taking input of the text for encryption
    char f; int init = 0;
    /* Key reading */
    string pathToFile = "C:/Users/Zeephremia/Desktop/a.txt";
    ifstream tfs(pathToFile);
    while (!tfs.eof())
    {
        tfs >> f;
        text[init] = f;
        init++;
        /* Key is read in such a way that each character is stored into the array */
    }

    tfs.close();
    //cout << text << endl;

    // Message is type casted
    unsigned char *plaintext = (unsigned char *)text;
    unsigned char ciphertext[128];


    /* Buffer for the decrypted text */
    int ciphertext_len;


    /* Encryption of the plaintext */
    ciphertext_len = encrypt(plaintext, strlen((char *)plaintext), key, iv, ciphertext);
    ciphertext[ciphertext_len] = '\0';
    cout << "\n\nCipher text is \n \n";
    cout << ciphertext;

    ofstream e;
    e.open("c:/users/zeephremia/desktop/b.txt");
    e << ciphertext;
    BIO_dump_fp(stdout, (const char *)ciphertext, ciphertext_len);
    return 0;

}

However when I try to decrypt it with THIS code,

#include <openssl/conf.h>
#include <openssl/evp.h>
#include <openssl/err.h>
#include <string.h>
#include <string>
#include <iostream>
#include <fstream>
#include <openssl/applink.c>

using namespace std;

void handleErrors(void)
{
    ERR_print_errors_fp(stderr);
    abort();
}

int decrypt(unsigned char *ciphertext, int ciphertext_len, unsigned char *key,
    unsigned char *iv, unsigned char *plaintext)
{
    EVP_CIPHER_CTX *ctx;

    int len;
    int plaintext_len;

    /* Create and initialise the context */
    if (!(ctx = EVP_CIPHER_CTX_new())) handleErrors();


    if (1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, key, iv))
        handleErrors();


    if (1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len))
        handleErrors();
    plaintext_len = len;

    if (1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) handleErrors();
    plaintext_len += len;

    /* Clean up */
    EVP_CIPHER_CTX_free(ctx);

    return plaintext_len;
}

using namespace std;
int main(void)
{
    /* A 256 bit key */
    unsigned char key[2000];
    memset(key, 0, sizeof(key));
    char s; int initializer = 0;
    /* Key reading */
    string path = "C:/openssl/mykey.pem";
    ifstream myfile(path);
    while (!myfile.eof())
    {
        myfile >> s;
        key[initializer] = s;
        initializer++;
        /* Key is read in such a way that each character is stored into the array */
    }

    // cout << key;

    myfile.close();


    /* A 128 bit IV */
    unsigned char *iv = (unsigned char *)"0123456789012345";
    /* Message to be encrypted */
    //unsigned char text[]="weufhskgwesjfho";
    char text[2000];
    memset(text, 0, sizeof(text));
    // Taking input of the text for encryption
    char f; int init = 0;
    /* Key reading */
    string pathToFile = "C:/Users/Zeephremia/Desktop/b.txt";
    ifstream tfs(pathToFile);
    while (!tfs.eof())
    {
        tfs >> f;
        text[init] = f;
        init++;
        /* Key is read in such a way that each character is stored into the array */
    }

    tfs.close();
    //cout << text << endl;

    // Message is type casted
    unsigned char *plaintext = (unsigned char *)text;
    unsigned char decryptedtext[2000];

    memset(decryptedtext, 0, sizeof(decryptedtext));
    int len = init-1;


    /* Buffer for the decrypted text */
    int decryptedtext_len;
    cout << "The encryption is: " << plaintext << endl;


    /*decrytption of the plaintext */
     decryptedtext_len = decrypt(plaintext, len, key, iv, decryptedtext);
     cout << decryptedtext; 
    system("pause");
    return 0;


}

it gives me this error

2332:Erorr:0605506D:lib(6) func(101) reason(100) evp_enc.c

along with this

Debug Error! Abort() has been cancelled.

From my little experimentation, I found that the error is on this line.

decryptedtext_len = decrypt(plaintext, len, key, iv, decryptedtext);

Any sort of help will be appreciated, thankyou very much. :)


Solution

  • It seems the ciphertext[plaintext_len] can't be decrypted.

    Dose ciphertext and plaintext_len are valid for aes decrypt?


    I have encountered a similar problem

    1. try to decrypt same file, on different host

    A. ubuntu14, openssl 1.0.1f

    chen@u14 $ md5sum hide.enc key.bin 
    51da135538878c53d0197485e0343f40  hide.enc
    bebbd6cf7cd090b5acd534646d85f487  key.bin
    chen@u14 $ openssl enc -d -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin
    bad decrypt
    139933588633248:error:06065064:digital envelope routines:EVP_DecryptFinal_ex:bad decrypt:evp_enc.c:539:
    // failed
    chen@u14 $ openssl version
    OpenSSL 1.0.1f 6 Jan 2014
    

    B. openwrt18, openssl 1.0.2p

    root@openwrt18# md5sum hide.enc key.bin 
    51da135538878c53d0197485e0343f40  hide.enc
    bebbd6cf7cd090b5acd534646d85f487  key.bin
    root@openwrt18# openssl enc -d -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin
    WARNING: can't open config file: /etc/ssl/openssl.cnf
    bad decrypt
    2013216028:error:06065064:lib(6):func(101):reason(100):NA:0:
    // failed
    root@openwrt18# openssl version
    WARNING: can't open config file: /etc/ssl/openssl.cnf
    OpenSSL 1.0.2p  14 Aug 2018
    

    C. ubuntu18, openssl 1.1.0

    chen@u18 $ md5sum hide.enc key.bin 
    51da135538878c53d0197485e0343f40  hide.enc
    bebbd6cf7cd090b5acd534646d85f487  key.bin
    chen@u18 $ openssl enc -d -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin
    // success
    chen@u18 $ openssl version
    OpenSSL 1.1.0g  2 Nov 2017
    

    2. some more research

    A. ubuntu 14

    openssl enc -d  -p -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin
    salt=6CA0C91549E1177C
    key=6E838B9ED5113E254020F895A419355F50F49245789662D5B9D9A89E8F6434DF
    iv =1A1CD428E24A8A4B25B1EC4A8ED6F136
    bad decrypt
    139658322110112:error:06065064:digital envelope routines:EVP_DecryptFinal_ex:bad decrypt:evp_enc.c:539:
    

    B. openwrt 18

    openssl enc -d -p -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin
    WARNING: can't open config file: /etc/ssl/openssl.cnf
    salt=6CA0C91549E1177C
    key=6E838B9ED5113E254020F895A419355F50F49245789662D5B9D9A89E8F6434DF
    iv =1A1CD428E24A8A4B25B1EC4A8ED6F136
    bad decrypt
    2013138204:error:06065064:lib(6):func(101):reason(100):NA:0:
    

    C. ubuntu 18

    openssl enc -d  -p -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin
    salt=6CA0C91549E1177C
    key=DD38E62703B2AF362362AED7EF64CB4268C053FC40C5F07EF085E014EEA5F27C
    iv =B15E1A3C723A0BCEAFF384ABD98AB81B
    

    // they got different key,iv, that's weird

    3. the result

    ubuntu 14, openssl 1.0.1f, success

    openssl enc -d  -p -aes-256-cbc -in hide.enc -out hide.txt -pass file:./key.bin -md sha256
    salt=6CA0C91549E1177C
    key=DD38E62703B2AF362362AED7EF64CB4268C053FC40C5F07EF085E014EEA5F27C
    iv =B15E1A3C723A0BCEAFF384ABD98AB81B
    

    finally, I recognize the default -md arg was different on openssl 1.1.0.

    the default values here:

    • ubuntu 14, openssl 1.0.2: md5
    • ubuntu 18, openssl 1.1.0: sha256

    default value seems from code source,

    /etc/ssl/openssl.cnf didn't found them.


    some thing maybe helpful.

    openssl enc -d  -aes-256-cbc -in hide.enc -out hide.txt -k ' '  -S 6CA0C91549E1177C -K DD38E62703B2AF362362AED7EF64CB4268C053FC40C5F07EF085E014EEA5F27C -iv B15E1A3C723A0BCEAFF384ABD98AB81B -p
    salt=6CA0C91549E1177C
    key=DD38E62703B2AF362362AED7EF64CB4268C053FC40C5F07EF085E014EEA5F27C
    iv =B15E1A3C723A0BCEAFF384ABD98AB81B
    

    // use openssl cmd do a aes-256-cbc decryption, and specify key,iv, salt
    // if not specify -k, -S seems not work

    full help

    openssl 1.1.0

    openssl enc --help
    Usage: enc [options]
    Valid options are:
     -help          Display this summary
     -ciphers       List ciphers
     -in infile     Input file
     -out outfile   Output file
     -pass val      Passphrase source
     -e             Encrypt
     -d             Decrypt
     -p             Print the iv/key
     -P             Print the iv/key and exit
     -v             Verbose output
     -nopad         Disable standard block padding
     -salt          Use salt in the KDF (default)
     -nosalt        Do not use salt in the KDF
     -debug         Print debug info
     -a             Base64 encode/decode, depending on encryption flag
     -base64        Same as option -a
     -A             Used with -[base64|a] to specify base64 buffer as a single line
     -bufsize val   Buffer size
     -k val         Passphrase
     -kfile infile  Read passphrase from file
     -K val         Raw key, in hex
     -S val         Salt, in hex
     -iv val        IV in hex
     -md val        Use specified digest to create a key from the passphrase
     -none          Don't encrypt
     -*             Any supported cipher
     -engine val    Use engine, possibly a hardware device
    

    openssl 1.0.1f

    openssl enc --help
    unknown option '--help'
    options are
    -in <file>     input file
    -out <file>    output file
    -pass <arg>    pass phrase source
    -e             encrypt
    -d             decrypt
    -a/-base64     base64 encode/decode, depending on encryption flag
    -k             passphrase is the next argument
    -kfile         passphrase is the first line of the file argument
    -md            the next argument is the md to use to create a key
                     from a passphrase.  One of md2, md5, sha or sha1
    -S             salt in hex is the next argument
    -K/-iv         key/iv in hex is the next argument
    -[pP]          print the iv/key (then exit if -P)
    -bufsize <n>   buffer size
    -nopad         disable standard block padding
    -engine e      use engine e, possibly a hardware device.
    Cipher Types
    -aes-128-cbc               -aes-128-cbc-hmac-sha1     -aes-128-cfb              
    -aes-128-cfb1              -aes-128-cfb8              -aes-128-ctr              
    -aes-128-ecb               -aes-128-gcm               -aes-128-ofb              
    -aes-128-xts               -aes-192-cbc               -aes-192-cfb              
    -aes-192-cfb1              -aes-192-cfb8              -aes-192-ctr              
    -aes-192-ecb               -aes-192-gcm               -aes-192-ofb              
    -aes-256-cbc               -aes-256-cbc-hmac-sha1     -aes-256-cfb              
    -aes-256-cfb1              -aes-256-cfb8              -aes-256-ctr              
    -aes-256-ecb               -aes-256-gcm               -aes-256-ofb              
    -aes-256-xts               -aes128                    -aes192                   
    -aes256                    -bf                        -bf-cbc                   
    -bf-cfb                    -bf-ecb                    -bf-ofb                   
    -blowfish                  -camellia-128-cbc          -camellia-128-cfb         
    -camellia-128-cfb1         -camellia-128-cfb8         -camellia-128-ecb         
    -camellia-128-ofb          -camellia-192-cbc          -camellia-192-cfb         
    -camellia-192-cfb1         -camellia-192-cfb8         -camellia-192-ecb         
    -camellia-192-ofb          -camellia-256-cbc          -camellia-256-cfb         
    -camellia-256-cfb1         -camellia-256-cfb8         -camellia-256-ecb         
    -camellia-256-ofb          -camellia128               -camellia192              
    -camellia256               -cast                      -cast-cbc                 
    -cast5-cbc                 -cast5-cfb                 -cast5-ecb                
    -cast5-ofb                 -des                       -des-cbc                  
    -des-cfb                   -des-cfb1                  -des-cfb8                 
    -des-ecb                   -des-ede                   -des-ede-cbc              
    -des-ede-cfb               -des-ede-ofb               -des-ede3                 
    -des-ede3-cbc              -des-ede3-cfb              -des-ede3-cfb1            
    -des-ede3-cfb8             -des-ede3-ofb              -des-ofb                  
    -des3                      -desx                      -desx-cbc                 
    -id-aes128-GCM             -id-aes192-GCM             -id-aes256-GCM            
    -rc2                       -rc2-40-cbc                -rc2-64-cbc               
    -rc2-cbc                   -rc2-cfb                   -rc2-ecb                  
    -rc2-ofb                   -rc4                       -rc4-40                   
    -rc4-hmac-md5              -seed                      -seed-cbc                 
    -seed-cfb                  -seed-ecb                  -seed-ofb