How would I properly include nasm functions in cpp project?

Question

I am trying to include a function I have coded in nasm into a project that is in c++.

When I try to include the function in the cpp file using the extern keyword and compile the project, linker (ld) fails with "undefined reference to `rnd_by_seed(unsigned long long)".

Almost as if there was missing some reference in cmakelists file.

cmake_minimum_required(VERSION 3.23)
project(App)

set(CMAKE_CXX_STANDARD 23)
#enable_language(ASM_NASM)
set(NASM_COMPILER nasm)
set(CMAKE_ASM_NASM_OBJECT_FORMAT elf64)
set(CMAKE_ASM_NASM_COMPILE_OBJECT "<CMAKE_ASM_NASM_COMPILER> <INCLUDES> <FLAGS>
-f ${CMAKE_ASM_NASM_OBJECT_FORMAT}  -o <OBJECT> <SOURCE>")

set(SOURCE_FILES main.cpp Tools/rnd_byseed.asm)
add_executable(App ${SOURCE_FILES})

here's the nasm function I am trying to execute:

global rnd_by_seed

;------------------------------------------
; return random number by seed value
; @param rdi seed value (unsigned 64bit integer)
; @return rax random number (64bit unsigned)
rnd_by_seed:
    push rbp
    mov rbp, rsp

    mov    eax, dword [rsp+8] 
    mov    ebx, 16807 
    mul    ebx
    mov    esi, edx 
    mov    edi, eax 
    mov    eax, dword [rsp+4] 
    mul    ebx 
    add    eax, esi 
    adc    edx, 0 

   
    shl    eax, 1
    rcl    edx, 1
    shr    eax, 1
    
    add    edx, edi 
    adc    eax, 0 
    xchg eax, edx
    
    test edx, 80000000h
    jz Store
    and    edx, 7fffffffh 
    add    eax, 1 
    adc    edx, 0

    Store:

   mov    dword [rsp+4], edx
   mov    dword [rsp+8], eax

   mov    rax, qword [rsp+8] 
   add rsp, 16
leave
ret

and main.cpp:

#include <iostream>

extern "C" unsigned long long rnd_by_seed(unsigned long long seed);

int main()
{
    std::cout << "Hello, World!" << std::endl;
    // call rnd_rnd_by_seed function
    auto val = rnd_by_seed(12);
    printf("val = %llu", val);
    return 0;
}

I tried to look at different methods for including nasm files into cmakelists file, got worse results. This is the only cmakelists file that does not fail when reloading the project.

Answer 1

You're almost there, just need to uncomment the enable_language(ASM_NASM) line.

enable_language(ASM_NASM)
set(CMAKE_ASM_NASM_OBJECT_FORMAT elf64)
set(CMAKE_ASM_NASM_COMPILE_OBJECT "<CMAKE_ASM_NASM_COMPILER> <INCLUDES> <FLAGS -f ${CMAKE_ASM_NASM_OBJECT_FORMAT} -o <OBJECT> <SOURCE>")

Or enable ASM_NASM at project level.

project(App C CXX ASM_NASM)

---

However, your assembly looks wrong - assuming x64 SysV ABI, I see at least the following issues:

• the first 64-bit argument is passed in via RDI, not stack
• the value of register RBX must be preserved
• writing to [rsp+4], [rsp+8] will damade the stack
• add rsp, 16 is bogus