Search code examples
c++crypto++

Invalid ciphertext when performing RSA decryption


I use the following code to encrypt and decrypt a string making use of Crypto++ 5.6.2 library

string to_BER(string spriv,bool b)    
{
    string HEADER, FOOTER;

    if(b)
    {
        HEADER = "-----BEGIN RSA PRIVATE KEY-----";
        FOOTER = "-----END RSA PRIVATE KEY-----";
    }
    else
    {
        HEADER = "-----BEGIN PUBLIC KEY-----";
        FOOTER = "-----END PUBLIC KEY-----";
    }

        size_t pos1, pos2;
        pos1 = spriv.find(HEADER);
        if(pos1 == string::npos)
            throw std::runtime_error("PEM header not found");

        pos2 = spriv.find(FOOTER, pos1+1);
        if(pos2 == string::npos)
            throw std::runtime_error("PEM footer not found");

        // Start position and length
        pos1 = pos1 + HEADER.length();
        pos2 = pos2 - pos1;
        string keystr = spriv.substr(pos1, pos2);

        return keystr;
}

string rsa_encrypt(const string &message, const string &public_key)
{    
    string keystr=to_BER(public_key,false);

    ByteQueue queue;
    Base64Decoder decoder;

    decoder.Attach(new Redirector(queue));
    decoder.Put((const byte*)keystr.data(), keystr.length());
    decoder.MessageEnd();

    RSAES_PKCS1v15_Encryptor e;
    e.AccessKey().Load(queue);

    bool key_ok = e.AccessKey().Validate(rng, 3);
    if(!key_ok)
    {
        printf("ERROR IN RSA KEY\n");
        return "";
    }

    string encrypted_data;

    StringSource ss1(message, true,
        new PK_EncryptorFilter(rng, e,
            new StringSink(encrypted_data)
       ) // PK_EncryptorFilter
    ); // StringSource

    return encrypted_data;
}

string rsa_decrypt(const string &message, const string &private_key,bool b_Base64decode)
{
    string keystr=to_BER(private_key,true);

    string decoded=message;

    if(b_Base64decode)
    {
        Base64Decoder decoder;

        decoder.Put( (byte*)message.data(), message.size() );
        decoder.MessageEnd();

        word64 size = decoder.MaxRetrievable();
        if(size && size <= SIZE_MAX)
        {
            decoded.resize(size);
            decoder.Get((byte*)decoded.data(), decoded.size());
        }
    }  

    RSAES_PKCS1v15_Decryptor pri( rng, 1024 );
    TransparentFilter privSink( new StringSink(keystr) );
    pri.DEREncode( privSink );
    privSink.MessageEnd();            

    string decrypted_data;
    try
    {

    StringSource ss( decoded, true,
            new PK_DecryptorFilter( rng, pri, new StringSink( decrypted_data )));
    }
    catch(Exception *e)
    {
        printf("ERROR DECRYPTING RSA\n");
        return "";
    }   

    return decrypted_data;
}

Then I use this code:

string enc=rsa_encrypt("hola mundo","-----BEGIN PUBLIC KEY----- MIGeMA0GCSqGSIb3DQEBAQUAA4GMADCBiAKBgHIiecdmRAaiTrCbbEOLBPh+fxp2KyGWyMTeWIj56NRk1CFLfBDT6vOWDceFpuTcrAFGR4Np4JLsssqM24F7bZCKv6eQCV8Xjv6GJiGofKEkI4b0zwHHEwq63p+9Rb5jYXlPZ7JIud7Yi96CHbg3foLQzZSSS9oFItGOFF0jDM7lAgMBAAE= -----END PUBLIC KEY-----");

string aaa=rsa_decrypt(
            /*"YTbXg1K4OlXGY6eaLuVTFZrN4qi1zg83h0PjeLd9F3Ge3gKUHsJpoE3iLv1+Gj/PepM8ehiilb5kphxCdcELjaYH9wwYHMpUZUQSRLQUTnofOboI6nfHaKnnNV28QMguM39q1hm7X1wNc69D8R+CkWLka2rQof+BXt+41ivnMj8=",*/
            enc
            ,"-----BEGIN RSA PRIVATE KEY----- MIICWwIBAAKBgHIiecdmRAaiTrCbbEOLBPh+fxp2KyGWyMTeWIj56NRk1CFLfBDT6vOWDceFpuTcrAFGR4Np4JLsssqM24F7bZCKv6eQCV8Xjv6GJiGofKEkI4b0zwHHEwq63p+9Rb5jYXlPZ7JIud7Yi96CHbg3foLQzZSSS9oFItGOFF0jDM7lAgMBAAECgYBxFL9bVHNprz4PtK3bbc2K9qmv6gxpxx88Dp/hdtm8NfoG0uclNRHALZeRa1Yjwo+Y46zPAwPCDkpGbLC+5S9zfBjtrx/+8zjTyMVb2CcGLfR0H2E/hcCjADXNxs+fmpB3+jyPhgH5ANaTmAXqGXOP56I0Fqo8xCfU/zQELCtzMQJBAN2Kq+9bQW3nKAAJEZqWQlAEjuBQfe1lrvLxc/AgVl9XLWrHre7HSlkyqcFemvqhzlZy1wz0Nv5VpOIGcAKefEMCQQCD4xSbkF1kzZyj8k6P3iUW6ezaK4krOZnpq/wDyOtj0DBAtLt3apyv3BUbe7AH1e8llJ5a8UYVHlsOdRUio+m3AkAB0LYR8wR5OxCn12sughavAyqMifxOKqwhT3sst4cdpuA3ZMV3FGj2jCS58eWBMjw3lx9N+t5MfTUpqPXX/6ZzAkBx9eTXqv3YXYZtb7GMxQI9c3Jy7k1/aS1iaXbA+nrTa5BWSRT30cqEduJSNiVcD/KuAWZ35KWPGATMUEUsAoCvAkEAuVg0OA6L3xJphKEbVYXvTwXrXcoVjjLDnNYjUJuNWrFFjBuNE4whWvdg76Panw3vMhgFc6yVr+VE5XOc7rXPvA== -----END RSA PRIVATE KEY-----"
            ,false
            );

The problem is that rsa_decrypt fails with a "invalid ciphertext" exception that isn't caught with the try/catch code. The program crashes. I would like to know what is wrong with my code and how can I avoid the program to crash even if the ciphertext is wrong. I have found similar threads (crypto++ RSA and "invalid ciphertext") but none of the solutions is valid for me: I don't see any problem with a 0 at the end of the string, I check that the keys are valid, I know that private and public keys are from the same key pair and they work with other RSA libraries.

This is the line were it crashes:

StringSource ss( decoded, true,
                new PK_DecryptorFilter( rng, pri, new StringSink( decrypted_data )));

Solution

  • The problem is that rsa_decrypt fails with a "invalid ciphertext" exception

    You create a new private key, and then encode it into keystr:

    RSAES_PKCS1v15_Decryptor pri( rng, 1024 );
    TransparentFilter privSink( new StringSink(keystr) );
    pri.DEREncode( privSink );
    

    Instead, you should use private_key (and not the new key) and do something like you did in the encryptor:

    string keystr=to_BER(private_key,true);
    
    ByteQueue queue;
    Base64Decoder decoder;
    
    decoder.Attach(new Redirector(queue));
    decoder.Put((const byte*)keystr.data(), keystr.length());
    decoder.MessageEnd();
    
    RSAES_PKCS1v15_Decryptor d;
    d.AccessKey().Load(queue);
    

    ... that isn't caught...

    Wrap everything in a try/catch block, and not just the PK_DecryptorFilter call.


    You can see the difference (or lackof) between the public and private keys with something like:

    try {
    
        string s1("-----BEGIN PUBLIC KEY-----\n"
                  "MIGeMA0GCSqGSIb3DQEBAQUAA4GMADCBiAKBgHIiecdmRAai"
                  "TrCbbEOLBPh+fxp2KyGWyMTeWIj56NRk1CFLfBDT6vOWDceF"
                  "puTcrAFGR4Np4JLsssqM24F7bZCKv6eQCV8Xjv6GJiGofKEk"
                  "I4b0zwHHEwq63p+9Rb5jYXlPZ7JIud7Yi96CHbg3foLQzZSS"
                  "S9oFItGOFF0jDM7lAgMBAAE="
                  "\n-----END PUBLIC KEY-----");
        string s2("-----BEGIN RSA PRIVATE KEY-----\n"
                  "MIICWwIBAAKBgHIiecdmRAaiTrCbbEOLBPh+fxp2KyGWyMTeWIj"
                  "56NRk1CFLfBDT6vOWDceFpuTcrAFGR4Np4JLsssqM24F7bZCKv6"
                  "eQCV8Xjv6GJiGofKEkI4b0zwHHEwq63p+9Rb5jYXlPZ7JIud7Yi"
                  "96CHbg3foLQzZSSS9oFItGOFF0jDM7lAgMBAAECgYBxFL9bVHNp"
                  "rz4PtK3bbc2K9qmv6gxpxx88Dp/hdtm8NfoG0uclNRHALZeRa1Y"
                  "jwo+Y46zPAwPCDkpGbLC+5S9zfBjtrx/+8zjTyMVb2CcGLfR0H2"
                  "E/hcCjADXNxs+fmpB3+jyPhgH5ANaTmAXqGXOP56I0Fqo8xCfU/"
                  "zQELCtzMQJBAN2Kq+9bQW3nKAAJEZqWQlAEjuBQfe1lrvLxc/Ag"
                  "Vl9XLWrHre7HSlkyqcFemvqhzlZy1wz0Nv5VpOIGcAKefEMCQQC"
                  "D4xSbkF1kzZyj8k6P3iUW6ezaK4krOZnpq/wDyOtj0DBAtLt3ap"
                  "yv3BUbe7AH1e8llJ5a8UYVHlsOdRUio+m3AkAB0LYR8wR5OxCn1"
                  "2sughavAyqMifxOKqwhT3sst4cdpuA3ZMV3FGj2jCS58eWBMjw3"
                  "lx9N+t5MfTUpqPXX/6ZzAkBx9eTXqv3YXYZtb7GMxQI9c3Jy7k1"
                  "/aS1iaXbA+nrTa5BWSRT30cqEduJSNiVcD/KuAWZ35KWPGATMUE"
                  "UsAoCvAkEAuVg0OA6L3xJphKEbVYXvTwXrXcoVjjLDnNYjUJuNW"
                  "rFFjBuNE4whWvdg76Panw3vMhgFc6yVr+VE5XOc7rXPvA=="
                  "\n-----END RSA PRIVATE KEY-----");
    
        ArraySource as1(s1, true), as2(s2, true);
    
        RSA::PublicKey k1;
        RSA::PrivateKey k2;
    
        PEM_Load(as1, k1);
        PEM_Load(as2, k2);
    
        AutoSeededRandomPool prng;
        k1.ThrowIfInvalid(prng, 3);
        k2.ThrowIfInvalid(prng, 3);
    
        Integer i1 = k1.GetModulus() - k2.GetModulus();
        Integer i2 = k1.GetPublicExponent() - k2.GetPublicExponent();
    
        cout << i1 << " " << i2 << endl;
    
    } catch (const Exception& ex) {
        cerr << ex.what() << endl;
    }
    

    Related, the encapsulation boundaries (-----BEGIN PUBLIC KEY----- and -----END PUBLIC KEY-----) should be on their own lines. Instead of a space, you should use a new line (\n). So something like:

    string enc=rsa_encrypt(
        "hola mundo",
        "-----BEGIN PUBLIC KEY-----\nMIGeMA0G...gMBAAE=\n-----END PUBLIC KEY-----"
        );
    

    And:

    string aaa=rsa_decrypt(
        enc,
        "-----BEGIN RSA PRIVATE KEY-----\nMIICWwIB...7rXPvA==\n-----END RSA PRIVATE KEY-----",
        false
        );
    

    Related, you should probably use RSAES_OAEP_SHA_Encryptor and RSAES_OAEP_SHA_Decryptor. An aaproachable treatment is available at A bad couple of years for the cryptographic token industry.