Problems with LD_PRELOAD and calloc() interposition for certain executables

Question

Relating to a previous question of mine

I've successfully interposed malloc, but calloc seems to be more problematic.

That is with certain hosts, calloc gets stuck in an infinite loop with a possible internal calloc call inside dlsym. However, a basic test host does not exhibit this behaviour, but my system's "ls" command does.

Here's my code:

// build with: g++ -O2 -Wall -fPIC -ldl -o libnano.so -shared Main.cc
#include <stdio.h>
#include <dlfcn.h>

bool gNanoUp = false;// global

// Function types
typedef void* (*MallocFn)(size_t size);
typedef void* (*CallocFn)(size_t elements, size_t size);

struct MemoryFunctions {
    MallocFn   mMalloc;
    CallocFn   mCalloc;
};

MemoryFunctions orgMemFuncs;

// Save original methods.
void __attribute__((constructor)) __nano_init(void) {
    fprintf(stderr, "NANO: init()\n");

    // Get address of original functions
    orgMemFuncs.mMalloc = (MallocFn)dlsym(RTLD_NEXT, "malloc");
    orgMemFuncs.mCalloc = (CallocFn)dlsym(RTLD_NEXT, "calloc");

    fprintf(stderr, "NANO: malloc() found @%p\n", orgMemFuncs.mMalloc);
    fprintf(stderr, "NANO: calloc() found @%p\n", orgMemFuncs.mCalloc);

    gNanoUp = true;
}

// replacement functions
extern "C" {
    void *malloc(size_t size) {
        if (!gNanoUp) __nano_init();
        return orgMemFuncs.mMalloc(size);
    }

    void* calloc(size_t elements, size_t size) {
        if (!gNanoUp) __nano_init();
        return orgMemFuncs.mCalloc(elements, size);
    }
}

Now, When I do the following, I get an infinite loop followed by a seg fault, eg:

% setenv LD_PRELOAD "./libnano.so"
% ls
...
NANO: init()
NANO: init()
NANO: init()
Segmentation fault (core dumped)

However if I comment out the calloc interposer, it almost seems to work:

% setenv LD_PRELOAD "./libnano.so"
% ls
NANO: init()
NANO: malloc() found @0x3b36274dc0
NANO: calloc() found @0x3b362749e0
NANO: init()
NANO: malloc() found @0x3b36274dc0
NANO: calloc() found @0x3b362749e0
<directory contents>
...

So somethings up with "ls" that means init() gets called twice.

EDIT Note that the following host program works correctly - init() is only called once, and calloc is successfully interposed, as you can see from the output.

// build with: g++ test.cc -o test
#include <stdio.h>
#include <stdlib.h>

int main(int argc, char* argv[]) {

    void* p = malloc(123);
    printf("HOST p=%p\n", p);
    free(p);

    char* c = new char;
    printf("HOST c=%p\n", c);
    delete c;

    void* ca = calloc(10,10);
    printf("HOST ca=%p\n", ca);
    free(ca);
}

% setenv LD_PRELOAD "./libnano.so"
% ./test 
NANO: init()
NANO: malloc() found @0x3b36274dc0
NANO: calloc() found @0x3b362749e0
HOST p=0x601010
HOST c=0x601010
HOST ca=0x601030

Answer 1

With regard to __nano_init() being called twice: You've declared the function as a constructor, so it's called when the library is loaded, and it's called a second time explicitly when your malloc() and calloc() implementations are first called. Pick one.

With regard to the calloc() interposer crashing your application: Some of the functions you're using, including dlsym() and fprintf(), may themselves be attempting to allocate memory, calling your interposer functions. Consider the consequences, and act accordingly.