d2i_RSA_PUBKEY always returns NULL

Question

I have to Work on some AIX related Openssl Bugs and tried to write some Testcode. In Short, i want to read in RSA Keyfiles in a Buffer and than get the Data out of the Buffer to do some checking. When i run the Testprogram i always get "Error : RSA is NULL", which means the Buffer is empty.

I created the Testkeys with this commandline:

openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -outform DER -out testprivate_key.pem
openssl rsa -in testprivate_key.pem -inform DER -outform DER -pubout -out testpublic_key.pem

I must admit i have no real clue about Openssl Programing so any Hints are highly welcome.

#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>

#include <openssl/pem.h>
#include <openssl/err.h>                                        /* ERR_* */
#include <openssl/rand.h>                                       /* RAND_* */
#include <openssl/bn.h>
#include <openssl/bio.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>


unsigned char buf[1024*10];

/* const char keyfiles[][32] = { "testpublic_key32.pem", "testpublic_key64.pem" }; */

int get_buffer () {
   RSA *pub_key = NULL;
   /* FILE *key_file; */
   /* key_file = fopen( keyfiles[0],"rt"); */
   FILE *key_file = fopen("testpublic_key.pem","r"); // read in key file
   if (!key_file)
   {
     printf("Failed to open key file\n");
    exit(1);
   }
   pub_key = PEM_read_RSA_PUBKEY(key_file,&pub_key,NULL,NULL); // fill buffer
   BIO *mem = BIO_new(BIO_s_mem());
   RSA_print(mem,pub_key,0);
   BIO_read(mem,buf,1024*10);

   BUF_MEM *bio_buf=NULL;
   BIO_get_mem_ptr(mem,&bio_buf);
   RSA_free(pub_key);
   BIO_free(mem);


}


int get_key(const unsigned char *buf, int len) {

    RSA *rsa = d2i_RSA_PUBKEY(NULL, &buf, len); // get data from buffer and do some checks
    if (rsa != NULL) {
        if (rsa->e != NULL) {
            printf("BN : <%s> (hex) -- <%s> (dec)\n", BN_bn2hex(rsa->e), BN_bn2dec(rsa->e));
            if (BN_is_odd(rsa->e) == 0) {
                printf("Error : RSA public exponent is even\n");
            } else {
                printf("RSA public exponent is OK.\n");
                return 0;
            }
        }
        RSA_free(rsa);
    }
       else {
        printf("Error : RSA is NULL\n");
    }
    return 1;
}

int main() {
    get_buffer();
    return get_key(buf, sizeof buf);
}

compile string:

gcc -lcrypto -o openssl_odd_even openssl_odd_even.c

/home/packagebuilder/test/openssl_bignum_issue # ./openssl_odd_even
Error : RSA is NULL

Answer 1

I'm not sure why you would want to throw the key around in some basic buffer when you can just as easily refer to the loaded key in the RSA structure directly. But assuming you have some reason, lets start with the basics and explain what's going on along the way.

Key Generation via CLI

You can generate the keypair in whatever form you want (PEM or DER). For us, we want PEM, so we have to use the genpkey interface. If we wanted DER we could just use the genrsa interface, which is much easier. Regardless, the syntax looks like this. Note the specific request to generate output in PEM form (thereby making us honest in our naming of a .pem file):

openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -outform PEM -out testprivate_key.pem

That should generate testprivate_key.pem, which will look like this:

-----BEGIN PRIVATE KEY-----
... base64 encoding of the key pair here ...
-----END PRIVATE KEY-----

Next, we want the public key from this key pair. We can use the openssl rsa CLI to get that as well.

openssl rsa -in testprivate_key.pem -inform PEM -pubout -out testpublic_key.pem -outform PEM

That should create testpublic_key.pem, and it should look like this:

-----BEGIN PUBLIC KEY-----
... bas64 encoding of public key here ...
-----END PUBLIC KEY-----

That takes care of the key file stuff. On to the code.

---

Reading RSA public key from PEM encoded file

Once we have the files, we can read either of them into a program using openssl libcrypto (for Windows its libeay32). This simple sample application reads the PEM-encoded public key from the current working directory, then stores it in a memory buffer in DER form, then dumps that memory buffer to stdout using BIO_dump (a very handy utility function from the bio api):

#include <stdio.h>
#include <stdlib.h>

#include <openssl/err.h>
#include <openssl/bio.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>

int main()
{
    OpenSSL_add_all_algorithms();
    OpenSSL_add_all_ciphers();

    // read the PEM from disk (assumes current working directory)
    FILE *fp = fopen("testpublic_key.pem", "r");
    if (fp == NULL)
    {
        perror("testpublic_key.pem");
        return EXIT_FAILURE;
    }

    // load from disk
    RSA *pub_key = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
    if (pub_key == NULL)
    {
        perror("Failed to load RSA public key from PEM file");
        return EXIT_FAILURE;
    }

    // no longer need this
    fclose(fp);

    // from here we can use pub_key however we want. in this case we're
    //  goin ot store it in a memory buffer in DER form.
    unsigned char buff[4*1024], *p = buff;
    int len = i2d_RSA_PUBKEY(pub_key, &p);
    if (len > 0)
    {
        // show the content of the buffer in the console using BIO-dump
        BIO* bio = BIO_new_fp(stdout, BIO_NOCLOSE);
        BIO_dump(bio, (const char*)buff, len);
        BIO_flush(bio);
        BIO_free(bio);
    }

    RSA_free(pub_key);

    return EXIT_SUCCESS;
}

Compiling that and running from the working directory where the key files we created earlier are locatd, will give us something similar to this:

Sample Output (varies, obviously)

0000 - 30 82 01 22 30 0d 06 09-2a 86 48 86 f7 0d 01 01   0.."0...*.H.....
0010 - 01 05 00 03 82 01 0f 00-30 82 01 0a 02 82 01 01   ........0.......
0020 - 00 9c b5 e2 ff c0 1b e8-c1 4d cc bb 76 c1 8b d6   .........M..v...
0030 - eb b6 ec 92 a3 e3 38 82-50 16 13 3d 2c bc ef 49   ......8.P..=,..I
0040 - 21 3c d6 83 ae 4d be b7-d5 7c 67 11 84 a4 ed 4e   !<...M...|g....N
0050 - 86 b4 c8 41 3e c4 70 e5-a1 cf 9d 13 26 6c bf f2   ...A>.p.....&l..
0060 - 5c 7e 4f 04 a6 0e e0 9d-90 55 87 67 e7 f8 58 62   \~O......U.g..Xb
0070 - a6 ff 85 a0 99 68 62 58-0b 02 66 74 3d f6 19 05   .....hbX..ft=...
0080 - 7d e3 51 fa b1 c2 db e2-f3 e8 fa f5 5f 36 95 67   }.Q........._6.g
0090 - cf 8f eb 32 7e 39 5d e1-37 30 57 5a 1d 25 9d fa   ...2~9].70WZ.%..
00a0 - ad 50 63 f5 23 14 b2 2a-de 10 7d b8 7e 83 2b b9   .Pc.#..*..}.~.+.
00b0 - 8c 8b aa 73 7b 4a 91 be-68 5d d8 ad d0 76 e0 de   ...s{J..h]...v..
00c0 - 15 bc c6 9a 77 f2 31 a9-11 e3 b7 83 ce ae e2 96   ....w.1.........
00d0 - 6b 9c 2b 20 b9 e5 d7 22-27 46 10 2b 91 5c a3 67   k.+ ..."'F.+.\.g
00e0 - 7e ea 8c d0 69 e1 06 0e-eb 1a a4 dd 22 b9 5e f8   ~...i.......".^.
00f0 - f1 8a db 73 86 57 f2 d5-d1 70 10 24 f7 08 1d 76   ...s.W...p.$...v
0100 - 14 2e de d0 47 95 ce ac-52 fc 4a 16 c5 19 29 cd   ....G...R.J...).
0110 - 94 40 f2 23 4d 63 03 5f-10 8a 21 6b 5d 5a 3c 30   .@.#Mc._..!k]Z<0
0120 - 1d 02 03 01 00 01                                 ......

Now you can use this buffer with your with your get_key function to examine the parts of the key itself (though why you would want to I have no clue, since you already had the RSA* earlier when it was loaded from the PEM). Modifying the source to add your get_key function and its examination of the public key gives us this:

#include <stdio.h>
#include <stdlib.h>

#include <openssl/err.h>
#include <openssl/bio.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>

int get_key(const unsigned char *buf, int len)
{
    int result = 1;

    RSA *rsa = d2i_RSA_PUBKEY(NULL, &buf, len);
    if (rsa != NULL)
    {
        if (rsa->e != NULL)
        {
            printf("BN : <%s> (hex) -- <%s> (dec)\n", BN_bn2hex(rsa->e), BN_bn2dec(rsa->e));
            if (BN_is_odd(rsa->e) == 0)
            {
                printf("Error : RSA public exponent is even\n");
            }
            else
            {
                printf("RSA public exponent is OK.\n");
                result = 0;
            }
        }
        RSA_free(rsa);
    }
    else
    {
        printf("Error : RSA is NULL\n");
    }
    return result;
}

int main()
{
    OpenSSL_add_all_algorithms();
    OpenSSL_add_all_ciphers();

    // read the PEM from disk (assumes current working directory)
    FILE *fp = fopen("testpublic_key.pem", "r");
    if (fp == NULL)
    {
        perror("testpublic_key.pem");
        return EXIT_FAILURE;
    }

    // load from disk
    RSA *pub_key = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
    if (pub_key == NULL)
    {
        perror("Failed to load RSA public key from PEM file");
        return EXIT_FAILURE;
    }

    // no longer need this
    fclose(fp);

    // from here we can use pub_key however we want. in this case we're
    //  goin ot store it in a memory buffer in DER form.
    unsigned char buff[4*1024], *p = buff;
    int len = i2d_RSA_PUBKEY(pub_key, &p);
    if (len > 0)
    {
        // show the content of the buffer in the console using BIO-dump
        BIO* bio = BIO_new_fp(stdout, BIO_NOCLOSE);
        BIO_dump(bio, (const char*)buff, len);
        BIO_flush(bio);
        BIO_free(bio);
    }

    RSA_free(pub_key);

    // run get_key
    printf("get_key returned %d\n", get_key(buff, len));

    return EXIT_SUCCESS;
}

Output

0000 - 30 82 01 22 30 0d 06 09-2a 86 48 86 f7 0d 01 01   0.."0...*.H.....
0010 - 01 05 00 03 82 01 0f 00-30 82 01 0a 02 82 01 01   ........0.......
0020 - 00 9c b5 e2 ff c0 1b e8-c1 4d cc bb 76 c1 8b d6   .........M..v...
0030 - eb b6 ec 92 a3 e3 38 82-50 16 13 3d 2c bc ef 49   ......8.P..=,..I
0040 - 21 3c d6 83 ae 4d be b7-d5 7c 67 11 84 a4 ed 4e   !<...M...|g....N
0050 - 86 b4 c8 41 3e c4 70 e5-a1 cf 9d 13 26 6c bf f2   ...A>.p.....&l..
0060 - 5c 7e 4f 04 a6 0e e0 9d-90 55 87 67 e7 f8 58 62   \~O......U.g..Xb
0070 - a6 ff 85 a0 99 68 62 58-0b 02 66 74 3d f6 19 05   .....hbX..ft=...
0080 - 7d e3 51 fa b1 c2 db e2-f3 e8 fa f5 5f 36 95 67   }.Q........._6.g
0090 - cf 8f eb 32 7e 39 5d e1-37 30 57 5a 1d 25 9d fa   ...2~9].70WZ.%..
00a0 - ad 50 63 f5 23 14 b2 2a-de 10 7d b8 7e 83 2b b9   .Pc.#..*..}.~.+.
00b0 - 8c 8b aa 73 7b 4a 91 be-68 5d d8 ad d0 76 e0 de   ...s{J..h]...v..
00c0 - 15 bc c6 9a 77 f2 31 a9-11 e3 b7 83 ce ae e2 96   ....w.1.........
00d0 - 6b 9c 2b 20 b9 e5 d7 22-27 46 10 2b 91 5c a3 67   k.+ ..."'F.+.\.g
00e0 - 7e ea 8c d0 69 e1 06 0e-eb 1a a4 dd 22 b9 5e f8   ~...i.......".^.
00f0 - f1 8a db 73 86 57 f2 d5-d1 70 10 24 f7 08 1d 76   ...s.W...p.$...v
0100 - 14 2e de d0 47 95 ce ac-52 fc 4a 16 c5 19 29 cd   ....G...R.J...).
0110 - 94 40 f2 23 4d 63 03 5f-10 8a 21 6b 5d 5a 3c 30   .@.#Mc._..!k]Z<0
0120 - 1d 02 03 01 00 01                                 ......
BN : <010001> (hex) -- <65537> (dec)
RSA public exponent is OK.
get_key returned 0

---

Removing Unneeded Code

The reality is, you don't need that buffer intermediate in the first place, and I hope that's obvious. Just load the properly crafted PEM from disk, and use the resulting RSA* from that to check the public exponent. The result is simply this:

#include <stdio.h>
#include <stdlib.h>

#include <openssl/err.h>
#include <openssl/bio.h>
#include <openssl/rsa.h>
#include <openssl/pem.h>

int check_key_exponent(RSA *rsa)
{
    int result = 1;

    if (rsa && rsa->e)
    {
        printf("BN : <%s> (hex) -- <%s> (dec)\n", BN_bn2hex(rsa->e), BN_bn2dec(rsa->e));
        if (BN_is_odd(rsa->e) == 0)
        {
            printf("Error : RSA public exponent is even\n");
        }
        else
        {
            printf("RSA public exponent is OK.\n");
            result = 0;
        }
    }
    else
    {
        printf("Error : RSA is NULL\n");
    }
    return result;
}

int main()
{
    OpenSSL_add_all_algorithms();
    OpenSSL_add_all_ciphers();

    // read the PEM from disk (assumes current working directory)
    FILE *fp = fopen("testpublic_key.pem", "r");
    if (fp == NULL)
    {
        perror("testpublic_key.pem");
        return EXIT_FAILURE;
    }

    // load from disk
    RSA *pub_key = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
    if (pub_key == NULL)
    {
        perror("Failed to load RSA public key from PEM file");
        return EXIT_FAILURE;
    }

    // no longer need this
    fclose(fp);

    // run get_key
    printf("check_key_exponent returned %d\n", check_key_exponent(pub_key));

    RSA_free(pub_key);

    return EXIT_SUCCESS;
}

Output

BN : <010001> (hex) -- <65537> (dec)
RSA public exponent is OK.
check_key_exponent returned 0