Search code examples
c++winapidllhookdetours

C++ hooking Member function and original function returns garbage value.


I want hook class member function(using dll injection). for now, hooking is success. but in hooking function, I coded that calling original function with return. as result, original function return garbage value. what wrong with my work?

target.cpp

#include <iostream>
#include <windows.h>

class Target
{
    private:
        int _member;

    public:
        Target()
        : _member(0)
        {
        }

        Target(int i)
        : _member(i)
        {
        }

        virtual ~Target()
        {
        }

        int getValue() // I want to hooking this function.
        {
            return _member;
        }
};

int main(int argc, char **argv)
{
    while(1){
        Sleep(10000);        
        Target objA, objB(7);
        std::cout << objA.getValue() << " " << objB.getValue() << std::endl;
    }
    return 0; 
}

injection.dll

#include <windows.h>
#include <detours.h>
#include <iostream>
#include <sstream>

#pragma comment(lib, "detours.lib")

static const int ADDRESS = 0x2180;
int (__thiscall * original_func)(void *);

int hookedFunction(void *obj)
{
    uintptr_t base = (uintptr_t)GetModuleHandle(0);
    std::cout << obj << " Hooked Obj Address.\n";
    int result = original_func(obj);
    std::cout << "original function returns " << result << '\n';
    return result; 
}

DWORD WINAPI Attach(LPVOID param)
{
    uintptr_t base = (uintptr_t)GetModuleHandle(0);

    original_func = (decltype(original_func)) (base + ADDRESS);

    OutputDebugString(TEXT("Attach approach!"));
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourAttach(&(PVOID&)original_func, hookedFunction);

    LONG lError = DetourTransactionCommit();
    if (lError != NO_ERROR) {
        OutputDebugString(TEXT("Attach fail!"));
    } else {
        OutputDebugString(TEXT("Attach Sucess!!"));
    }
    return 0;
}

DWORD WINAPI Detach(LPVOID param)
{
    OutputDebugString(TEXT("Detach approach!"));
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourDetach(&(PVOID&)original_func, hookedFunction);

    LONG lError = DetourTransactionCommit();
    if (lError != NO_ERROR) {
        OutputDebugString(TEXT("Detach fail!"));
    } else {
        OutputDebugString(TEXT("Detach Sucess!!"));
    }
    return 0;
}

BOOL APIENTRY DllMain(HANDLE hModule, DWORD dwReason, LPVOID lpReserved)
{
    switch(dwReason)
    {
        case DLL_PROCESS_ATTACH:
        {
            CreateThread(0, 0, Attach, hModule, 0, 0); 
            break;
        }
        case DLL_PROCESS_DETACH:
        {
            CreateThread(0, 0, Detach, hModule, 0, 0); 
            break;
        }
    }
    return TRUE;
}

console output of target.cpp program.

0 7
0 7
0 7
0 7
0 7
0 7
0 7 <-- (DLL injection!)
000D1069 Hooked Obj Address.
original function returns 843966720 <-- why not 0 or 7?
000D8B30 Hooked Obj Address.
original function returns 890668
890668 843966720
000D1069 Hooked Obj Address.
original function returns 843966720
000D8B30 Hooked Obj Address.
original function returns 890668

as you see in console output, original function returns garbage value. why original function returns garbage value?


Solution

  • From Using Detours:

    For proper interception the target function, detour function, and the target pointer must have exactly the same call signature including number of arguments and calling convention. Using the same calling convention insures that registers will be properly preserved and that the stack will be properly aligned between detour and target functions

    int hookedFunction(void *obj) does not match calling convention of __thiscall.

    There may be other errors, but that is an obvious one.