Error when trying to RSA-decrypt a parameter string with libcrypto

Question

I am a beginner with the libcrypto library. I am trying to give an encrypted string to a function, decrypt it and return it decrypted. The string is encoded with my public key which which is 4096-bit sized.

char* decodeStr(const char* str, const size_t sizeStr)
{
    puts("Starting");
    FILE* file = fopen(PRIVATE_KEY_PATH, "r");
    if (file == NULL)
    {
        perror("Error while trying to access to Presto's private key.\n");
        return NULL;
    }

    RSA *privateKey = RSA_new();
    privateKey = PEM_read_RSAPrivateKey(file, &privateKey, NULL, NULL);
    if (privateKey == NULL)
    {
        fprintf(stderr, "Error loading RSA private key.\n");
        ERR_print_errors_fp(stderr);
        return NULL;
    } 
    char* res = malloc(sizeStr);
    if (res == NULL)
    {
        perror("Memory allocating error ");
        return NULL;
    }
    const int sizeDecoded = RSA_private_decrypt(sizeStr, str, res, privateKey, RSA_PKCS1_PADDING);
    if (sizeDecoded == -1)
    {
        fprintf(stderr, "Error while decoding RSA-encoded wrapping key.\n");
        ERR_print_errors_fp(stderr);
        return NULL;
    }
    if ((res = realloc(res, (size_t)sizeDecoded)) == NULL)
    {
        perror("Memory allocating error ");
        return NULL;
    }
    return res;

}

The following code outputs :

Starting
Error while decoding RSA-encoded wrapping key.
6928:error;04069506C:lib<4>:func<101>:reason<108>:.\crypto\rsa\rsa_eay.c:518:

Since the error is unknown and I haven't been able to find any information about it on the net, and moreover I am a beginner with libcrypto, does strneed to be in a certain format ?

Obivously it is this which breaks the program but I can't be sure of it, neither do I know how to fix this.

const int sizeDecoded = RSA_private_decrypt(sizeStr, str, res, privateKey, RSA_PKCS1_PADDING);

EDIT : I have been working with a client which provides me those encoded dat for me to decrypt them. I don't know how they are processed exactly. Unfortunately, the encoded strings are even more sensitive than the private key itself, so I can't share it. It looks like 0c79cc00deb89a614db6ebe42be748219089fb5356 but with 1024 characters.

Answer 1

The problem

Your encoded strings

0c79cc00deb89a614db6ebe42be748219089fb5356 but with thousands of characters.

look like hexadecimal representation of the encoded message. I think you really need to convert the message to a raw data, converting hex to bytes.

Your strings are composed by 1024 (hex) chars => 512 bytes (you need two hex digits to represent one byte) => 4096 bits, which is equal to the key length.

Try this simple function.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>

void hex_2_bytes(uint8_t *des, char *source, size_t size);

int main()
{
                char *str = "0c79cc00deb89a614db6ebe42be748219089fb5356";
                uint8_t *dest = (uint8_t *) malloc(sizeof(uint8_t) * strlen(str) / 2);
                hex_2_bytes(dest, str, strlen(str));
                for(int i = 0; i < strlen(str) / 2; i++) {
                                printf(" %02X ", dest[i]);
                }
                return 0;
}

void hex_2_bytes(uint8_t *des, char *source, size_t size) {
                for(int i = 0; i < size - 1; i+=2) {
                                sscanf(source+i, "%02x", des + (i/2));
                }
}

What is raw data? Your encrypted message is a string, which is an array of characters. I see only digits (from 0 to 9) and the letters from a to f, which make me guess that your original message (which is, originally a binary string, raw data) has been represented as a very long number using hexadecimal digits. You need to convert it back.

If you have a message composed by n bytes, stored as a binary string (raw data), you will have an hexadecimal representation, stored as a text, which is 2*n bytes long.

Example:

uint8_t raw_data_message[] = {0x3c, 0x2f};  // 2 bytes
char hex_representation_as_string_message[] = {'3', 'c', '2', 'f'}; // 4 bytes

Your messages a like hex_representation_as_string_message, but you need them as raw_data_message.

The solution

Here's a fully working example:

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <openssl/pem.h>
#include <openssl/err.h>

#define PRIVATE_KEY_PATH ".ssh/id_rsa"

/*
 * Generation of the encrypted message:
 * echo "Hello, this is a super secret message" | openssl rsautl -encrypt -pubin -inkey ~/.ssh/id_rsa.pub.pem >message.encrypted
 *
 * Resulting encrypted message in hex representation:
 * 917ab2ebd663bba1dcd0f22aef98b93b039f001e19c997f015d90eaaf35731eb1895c13dfb08250aa28a9dac4f3e1b5fefc53926d3f6422f8055124bb15c24e2b73645dc61f29486deaf278705987738e916a6288531aa923dff15b667dabf4465506e1ee68e6f27a06e3fb4f5040a7775ce69ba10ec337f5bc5ef45969a8fe7c672c9b51243296677385f1b613f4f3edceef620f6ab5dcadec5034c330331bf8e8c3b42554f01c6cf1c0dbc58f23c8f0068e750fc4bb97636b2b3455e7f3932ab9559ff4de5bfc6769bbafefec722441458066ab4a6fdfe99c78bfdd5c1851d411a451925c5ad7ecb0c93618304ae6bc5402193f58af6e6a65208075be35a00
 * (converted with: hexdump -ve '1/1 "%.2x"' message.encrypted)
 */

void hex_2_bytes (uint8_t *des, char *source, size_t size);

int decode (uint8_t *dest, const uint8_t *src, const size_t size);

int main (int argc, char **argv) {
    // Reading the encrypted message in hex representation
    FILE *file = fopen(argv[1], "r");
    printf("%s\n", argv[1]);
    if (file == NULL) {
        perror("Error while trying to access to the encrypted message"
                       ".\n");
        return -1;
    }
    fseek(file, 0, SEEK_END);
    long fsize = ftell(file);
    fseek(file, 0, SEEK_SET);
    char *hex_repr = malloc(fsize);
    fread(hex_repr, fsize, 1, file);
    fclose(file);

    printf("Hexadecimal representation of the encrypted message\n");
    for (int i = 0; i < fsize; ++i) {
        printf("%c", hex_repr[i]);
    }
    printf("\nSize: %d\n", fsize);

    // Converting to raw data
    size_t raw_data_size = fsize / 2;
    uint8_t *raw_data = (uint8_t *) malloc(
            raw_data_size * sizeof(uint8_t));
    hex_2_bytes(raw_data, hex_repr, (size_t) fsize);
    printf("Raw encrypted message\n");
    for (int i = 0; i < raw_data_size; ++i) {
        printf("%02X", raw_data[i]);
    }
    printf("\nSize: %d\n", raw_data_size);

    // Decryption
    char *res = malloc(raw_data_size * sizeof(char));
    if (res == NULL) {
        perror("Memory allocating error ");
        return -1;
    }
    int msg_size = decode(res, raw_data, raw_data_size);
    printf("Decrypted message:\n");
    for (int j = 0; j < msg_size; ++j) {
        printf("%c", res[j]);
    }
    printf("\nSize: %d\n", msg_size);

    return 0;
}

void hex_2_bytes (uint8_t *des, char *source, size_t size) {
    for (int i = 0; i < size - 1; i += 2) {
        sscanf(source + i, "%02x", des + (i / 2));
    }
}

int decode (uint8_t *res, const uint8_t *src, const size_t size) {
    puts("Starting");
    FILE *file = fopen(PRIVATE_KEY_PATH, "r");
    if (file == NULL) {
        perror("Error while trying to access to Presto's private key.\n");
        return -1;
    }

    RSA *privateKey = RSA_new();
    privateKey = PEM_read_RSAPrivateKey(file, &privateKey, NULL, NULL);
    if (privateKey == NULL) {
        fprintf(stderr, "Error loading RSA private key.\n");
        ERR_print_errors_fp(stderr);
        return -1;
    }
    const int sizeDecoded = RSA_private_decrypt(size, src, res,
                                                privateKey,
                                                RSA_PKCS1_PADDING);
    if (sizeDecoded == -1) {
        fprintf(stderr,
                "Error while decoding RSA-encoded wrapping key.\n");
        ERR_print_errors_fp(stderr);
        return -1;
    }
    return sizeDecoded;
}

And the results (as a proof of work):

Note that I'm using 2048 bit rsa keys.