Call functions from x86_64 assembly

Question

I am trying to create my own JIT and so far managed to run very simple assembly code (in machine-code), but having trouble figuring out how to call functions this way. In Visual Studio I can see functions in disassembly window.

Another related question is how do I call Win32 MessageBox() in machine-code?

Next question is how do I call external DLL/LIB functions in this manner?

Also is there any books or tutorials which could teach me further in this subject? I have tried to search for it but get results like .NET, JVM and LLVM which I think is not really what I am looking for.

Here is a simplified version of the code that I am working on:

#include <iostream>
#include <Windows.h>

int main(int argc, char* argv[])
{
    // b8 03 00 00 00 83 c0 02 c3
    unsigned char code[] = {
        0xb8,                   // mov eax, 3
        0x03, 0x00, 0x00, 0x00, // 3 (32 bit)
        0x83,                   // add eax, 2 // 0x83 = add,
        0xc0,                   // ModR/M with immediate 8 bit value
        0x02,                   // 2 (8 bit)
        0xc3                    // ret
    };

    void* mem = VirtualAlloc(0, sizeof(code), MEM_COMMIT, PAGE_EXECUTE_READWRITE);

    memcpy(mem, code, sizeof(code));

    DWORD old;
    VirtualProtect(mem, sizeof(mem), PAGE_EXECUTE_READ, &old);

    int(*func)() = reinterpret_cast<int(*)()>(mem);

    printf("Number is %d\n", func());

    VirtualFree(mem, 0, MEM_RELEASE);

    return 0;
}

Is it possible to have the JIT assembly code to call a C++ function?

Before this project I made a byte-code interpreter in C++, but I wasn't really happy with the speed when comparing it to equivalent test program in C#. C# was roughly 25x times faster. So I stumbled on something called JIT to make it faster. So I hope you all can see where I am taking this JIT project. And maybe if possible make it handle GUI.

Answer 1

You can probably find some tutorials about writing a compiler/linker. It may help with implementing/calling dynamic libraries.

I'm not sure what you exactly mean by calling C++ functions. Anyway I wrote the following demo program that you can take a look and see if it helps at all.

#include <Windows.h>
#include <iostream>


using namespace std;

__int64 sub(__int64 a, __int64 b)
{
    return a - b;
}

int main(int argc, char **argv)
{
    char code[] =
    {
        0x48, 0x89, 0xC8,           // mov rax, rcx
        0xC3,                       // ret

        0x48, 0x83, 0xEC, 0x20,     // sub rsp, 0x20
        0xFF, 0xD0,                 // call rax
        0x48, 0x83, 0xC4, 0x20,     // add rsp, 0x20
        0xC3                        // ret
    };


    char *mem = static_cast<char *>(VirtualAlloc(0, sizeof(code), MEM_COMMIT, PAGE_EXECUTE_READWRITE));

    MoveMemory(mem, code, sizeof(code));

    auto setFunc = reinterpret_cast<void *(*)(void *)>(mem);
    auto callFunc = reinterpret_cast<__int64 (*)(__int64, __int64)>(mem + 4);

    setFunc(sub);
    __int64 r = callFunc(0, 1);
    cout << "r = " << r << endl;

    VirtualFree(mem, 0, MEM_RELEASE);


    cin.ignore();
    return 0;
}