There is a cleanup command in openssl engine API called ENGINE_finish(e), which calls a "finish" command implemented and registered in the engine e.
The engine's "finish" command will be called only if a reference count to the engine is equal to 1. I don't understand why in my case it's 3. When you use libcurl, the ref count will also increase by one after you call a method POST and it will be 4.
To make sure that the "finish" command is called, I need to run an ugly loop like below and I don't know if it can cause a potential damage to the other components:
while (e->funct_ref) {
ENGINE_finish(e);
}
I've stripped out my initial code to demonstrate the error and made it really simple. The code of the "dummy" engine and the client's code are below. Please let me know how to make ENGINE_finish working without the ugly code above and why ref count is equal to 3 while it should be 1.
Dummy engine:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <openssl/ssl.h>
#include <openssl/crypto.h>
#include <openssl/ecdsa.h>
#include <openssl/engine.h>
#include <openssl/eng_int.h>
static int dummy_destroy(ENGINE *e);
static int dummy_init(ENGINE *e);
static int dummy_finish(ENGINE *e);
static int dummy_ctrl(ENGINE *e, int cmd, long i, void *p,
void (*f) ());
static int dummy_rand_bytes(unsigned char *buf, int num);
static void dummy_rand_seed(const void *buf, int num);
static void dummy_rand_seed(const void *buf, int num) {
}
static void dummy_rand_cleanup();
static void dummy_rand_cleanup(){
}
static void dummy_rand_add(const void *buf, int num, double entropy);
static void dummy_rand_add(const void *buf, int num, double entropy){
}
static int dummy_rand_status(){
fprintf(stderr, "dummy_rand_status\n");
return 1;
}
# define DUMMY_CMD_SO_PATH ENGINE_CMD_BASE
static const ENGINE_CMD_DEFN dummy_cmd_defns[] = {
{DUMMY_CMD_SO_PATH,
"SO_PATH",
"Specifies the path to the dummy shared library",
ENGINE_CMD_FLAG_STRING},
{0, NULL, NULL, 0}
};
static char *my_prog = "dummy";
static EC_KEY *ec_key=NULL;
static char *key_file = "path-to-ec-key";
int dummy_ecdsa_sign_setup (EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp,
BIGNUM **rp);
static ECDSA_SIG *dummy_ecdsa_sign (const unsigned char *dgst, int dgst_len,
const BIGNUM *kinv, const BIGNUM *rp,
EC_KEY *in_eckey);
int dummy_ecdsa_do_verify (const unsigned char *digest, int digest_len,
const ECDSA_SIG *ecdsa_sig, EC_KEY *eckey);
static ECDSA_METHOD dummy_ecdsa = {
"dummy ECDSA method",
dummy_ecdsa_sign,
dummy_ecdsa_sign_setup,
dummy_ecdsa_do_verify,
0, /* flags */
NULL /* app_data */
};
static RAND_METHOD dummy_rand = {
/* "Cluster Labs RAND method", */
dummy_rand_seed, /* seed */
dummy_rand_bytes, /* bytes */
dummy_rand_cleanup, /* cleanup */
dummy_rand_add, /* add */
dummy_rand_bytes, /* pseudorand */
dummy_rand_status, /* status */
};
static const char *engine_dummy_id = "dummy";
static const char *engine_dummy_name =
"DUMMY Secure Element Support";
/* engine implementation */
/* ---------------------*/
static int bind_helper(ENGINE *e)
{
if (!ENGINE_set_id(e, engine_dummy_id) ||
!ENGINE_set_name(e, engine_dummy_name) ||
!ENGINE_set_RAND(e, &dummy_rand) ||
!ENGINE_set_ECDSA(e, &dummy_ecdsa) ||
!ENGINE_set_destroy_function(e, dummy_destroy) ||
!ENGINE_set_init_function(e, dummy_init) ||
!ENGINE_set_finish_function(e, dummy_finish) ||
!ENGINE_set_ctrl_function(e, dummy_ctrl) ||
!ENGINE_set_cmd_defns(e, dummy_cmd_defns))
return 0;
return 1;
}
static int dummy_destroy(ENGINE *e)
{
fprintf(stderr, "%s: DESTROYED\n", my_prog);
return 1;
}
int dummy_init(ENGINE *e)
{
fprintf(stderr, "%s: INIT ref cnt: %d\n", my_prog, e->funct_ref);
FILE *fp = fopen(key_file, "r");
if (!fp) {
fprintf(stderr,"%s: Can't open %s\n", my_prog, key_file);
return 0;
}
EVP_PKEY *pkey = PEM_read_PrivateKey(fp, NULL, 0, NULL);
if (pkey) {
ec_key = EVP_PKEY_get1_EC_KEY(pkey);
fprintf(stderr,"%s: Got ec key %p\n", my_prog, ec_key);
EVP_PKEY_free (pkey);
}
fprintf(stderr, "%s: INIT ENDS ref cnt: %d\n", my_prog, e->funct_ref);
return 1;
}
static int dummy_finish(ENGINE *e)
{
fprintf(stderr, "%s: FINISHED\n", my_prog);
return (1);
}
static int dummy_ctrl(ENGINE *e, int cmd, long i, void *p,
void (*f) ())
{
fprintf(stderr, "dummy_trl cmd %d\n", cmd);
switch (cmd) {
case DUMMY_CMD_SO_PATH:
if (p == NULL) {
return 0;
}
return 1;
default:
break;
}
return 0;
}
static int dummy_rand_bytes(unsigned char *buf, int num)
{
fprintf(stderr, "dummy_rand num = %d \n", num);
fflush(stderr);
for (int i=0; i < num; i++) {
buf[i]=rand();
}
return 1;
}
int dummy_ecdsa_sign_setup (EC_KEY *eckey, BN_CTX *ctx_in, BIGNUM **kinvp,
BIGNUM **rp)
{
return 1;
}
static ECDSA_SIG *dummy_ecdsa_sign (const unsigned char *dgst, int dgst_len,
const BIGNUM *kinv, const BIGNUM *rp,
EC_KEY *in_eckey) {
printf("dummy engine ecdsa sign digest \n");
if (ec_key != NULL) {
fprintf(stderr, "%s: got private ec_key\n", my_prog);
in_eckey = ec_key;
}
return ECDSA_do_sign_ex(dgst, dgst_len, kinv, rp, in_eckey);
}
int dummy_ecdsa_do_verify (const unsigned char *dgst, int dgst_len,
const ECDSA_SIG *ecdsa_sig, EC_KEY *eckey) {
printf("dummy engine verifying function\n");
return ECDSA_do_verify(dgst, dgst_len, ecdsa_sig, eckey);
}
# ifdef ENGINE_DYNAMIC_SUPPORT
static int bind_fn(ENGINE *e, const char *id)
{
fprintf(stderr, "bind_fn DUMMY\n");
if (id && (strcmp(id, engine_dummy_id) != 0)) {
fprintf(stderr, "bind_fn return(0) first\n");
return 0;
}
if (!bind_helper(e)) {
fprintf(stderr, "bind_fn return(1) first\n");
return 0;
}
fprintf(stderr, "bind_fn return(1) %d\n", e->funct_ref);
return 1;
}
IMPLEMENT_DYNAMIC_CHECK_FN()
IMPLEMENT_DYNAMIC_BIND_FN(bind_fn)
# endif /* ENGINE_DYNAMIC_SUPPORT */
Client's code:
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <openssl/engine.h>
#include <openssl/eng_int.h>
static char *myprog = "engtest";
ENGINE *set_engine (const char * name) {
ENGINE *e = NULL;
ENGINE_load_builtin_engines();
e = ENGINE_by_id(name);
if(!e || !ENGINE_init(e)) {
#ifdef DEBUG
fprintf(stderr,"%s: can't find or init engine %s %p\n", myprog, name, e);
#endif
if (e)
ENGINE_free(e);
return NULL;
}
if (!ENGINE_set_default(e, ENGINE_METHOD_ALL)){
#ifdef DEBUG
fprintf(stderr,"%s: can't set engine %s as default %p\n", myprog, name, e);
#endif
ENGINE_finish(e);
ENGINE_free(e);
return NULL;
}
fprintf(stderr, "%s: set eng %s %p %p, ref cnt = %d \n", myprog, name, e, e->finish, e->funct_ref);
return e;
}
int unset_engine (ENGINE * e) {
if (!e)
return 0;
ENGINE_set_RAND(e, NULL);
ENGINE_set_ECDSA(e, NULL);
ENGINE_set_default(NULL, 0);
fprintf(stderr, "%s: unset %p, ref cnt = %d\n", myprog, e, e->funct_ref);
while (e->funct_ref) {
ENGINE_finish(e);
}
ENGINE_free(e);
return 1;
}
int main (int argc, char *argv[]) {
ENGINE *e = set_engine("dummy");
unset_engine(e);
return 0;
}
The output looks like follows. It's 0 at init and then gets bumped to 3 somehow.
bind_fn DUMMY
bind_fn return(1) 0
dummy: INIT ref cnt: 0
dummy: Got ec key 0x7f8199c04b70
dummy: INIT ENDS ref cnt: 0
engtest: set eng dummy 0x7f8199c04180 0x101bd69a0, ref cnt = 3
engtest: unset 0x7f8199c04180, ref cnt =3
dummy: FINISHED
The entire mechanism of cleaning up an engine (and certain other types of data) in OpenSSL seems rather brittle to me and your problem looks familiar. In addition to memory that does not get freed, it is easy to run into crashes if certain clean-up calls are not executed in the proper order. I have not taken the effort to reproduce your particular issue, but here is a snippet of code that I use to cleanly release my engine, so you can give that a try:
OBJ_cleanup();
EVP_cleanup();
if (NULL != S_engine) {
ENGINE_unregister_ciphers(S_engine);
ENGINE_unregister_digests(S_engine);
ENGINE_unregister_ECDH(S_engine);
ENGINE_unregister_ECDSA(S_engine);
ENGINE_unregister_RAND(S_engine);
ENGINE_finish(S_engine);
ENGINE_remove(S_engine);
S_engine = NULL;
}
ENGINE_cleanup();