OpenSSL RSA_sign C++ - Different sign than the command line

Question

I am trying to develop a Digital Signature tool using OpenSSL's libs but the created sign from my code is different than the one I get from the command line.

I checked and validated that the SHA256 digest is correct.

This is an example of the command I use to validate the signatures:

openssl rsautl -sign -inkey privateKey.pem -in hash.txt > signature

Based on my code below, is there something obvious that I am doing wrong?

#include <iostream>
#include <string>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <string.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/sha.h>
#include <openssl/rsa.h>
using namespace std;

RSA * generateKeys()
{
    RSA *rsa = RSA_new();
    BIGNUM *bignum = NULL;
    bignum = BN_new();
    BN_set_word(bignum, RSA_F4);

    FILE *file;
    bool exists;

    if ((file = fopen("privateKey.pem", "r")))
    {
        exists = true;
        fclose(file);
    }
    if (!exists)
    {
        FILE *pkey = fopen("privateKey.pem", "wb");

        RSA_generate_key_ex(rsa, 2048, bignum, NULL);
        PEM_write_RSAPrivateKey(pkey, rsa, NULL, NULL, 0, NULL, NULL);

        fclose(pkey);

    }else{
        FILE *pkey = fopen("privateKey.pem", "rb");

        PEM_read_RSAPrivateKey(
            pkey,
            &rsa,
            NULL,
            NULL);
        
        fclose(pkey);
    }

    return rsa;
}

// Sign document
void sign(const char *filepath)
{
    FILE *file = fopen(filepath, "r");

    // SHA256 digest
    unsigned char hash[SHA256_DIGEST_LENGTH];
    SHA256_CTX sha256;
    SHA256_Init(&sha256);

    char *buffer[1024];
    int bytesRead = 0;
    while ((bytesRead = fread(buffer, 1, 1024, file)))
    {
        SHA256_Update(&sha256, buffer, bytesRead);
    }
    fclose(file);

    SHA256_Final(hash, &sha256);

    // Retrieve keys (create keys if it does not exist)
    RSA *rsa = generateKeys();

    // Sign
    const int size = RSA_size(rsa);
    unsigned char *sign = new unsigned char[size];
    unsigned int outlen = 0;
    RSA_sign(NID_sha256, hash, SHA256_DIGEST_LENGTH, sign, &outlen, rsa);

    FILE *signedDoc = fopen("signedDocument.signed","wb");

    fputs((const char *) sign,signedDoc);
    fclose(signedDoc);
}

// Validate document
// int validate(FILE *file){
//     return 0;
// }

int main(int argc, char *argv[])
{

    // Input validation
    if (argc == 1)
    {
        cout << "Usage: sign -sv document" << endl
             << "-s sign document" << endl
             << "-v validate document signature" << endl;
        return 0;
    }
    if (argc > 3 || (string(argv[1]) != "-s" && string(argv[1]) != "-v"))
    {
        cout << "Error: Wrong arguments given." << endl
             << "sign for help" << endl;
        return -1;
    }

    FILE *document = fopen(argv[2], "r");
    if (document == NULL)
    {
        cout << "File open error.." << endl;
        return -1;
    }

    // Sign document
    if ((string(argv[1]) == "-s"))
    {
        sign(argv[2]);
    }

    // // Validate signature
    // if((string(argv[1]) == "-v")){
    //     validate(document);
    // }
}

And these are the flags I use to compile it:

-Wall -Werror -Wextra -pedantic -lcrypto -std=c++11 -g

Thanks in advance.

Answer 1

I have implemented the RSA_verify function and based on the sign my code does, RSA_verify works and it returns that it is valid. I have also added the code to save the public key as well.

My best guess is that my code doesn't represent exactly the command.