RegOpenKeyEx returns error 87 in MASM64

Question

I am writing small application in MASM64 (ML64.exe). One of the functions is setting value in registry. I started with opening the key but there is an error.

Function RegOpenKeyEx returns error 87, and GetLastError returns 0x36B7.

My code:

extrn ExitProcess : proc
extrn MessageBoxA : proc
extrn RegOpenKeyExA : proc
extrn RegSetValueExA : proc
extrn RegCloseKey : proc
extrn GetLastError : proc

.const
HKEY_CURRENT_USER equ 80000001h
KEY_SET_VALUE equ 2
KEY_CREATE_SUB_KEY equ 4
KEY_WOW64_64KEY equ 0100h
KEY_WRITE equ 00020006h
REG_SZ equ 00000001h

.data
hReg dq 0
szRegKeyName db "Software\Microsoft\Windows\CurrentVersion\Run",0

.code

    Main proc
        sub rsp, 30h
        lea rax, hReg
        push rax
        mov r9, KEY_SET_VALUE+KEY_CREATE_SUB_KEY+KEY_WOW64_64KEY
        mov r8, 0
        lea rdx, szRegKeyName
        mov rcx, HKEY_CURRENT_USER
        call RegOpenKeyExA ;returns 87
        call GetLastError ;returns 0x36B7
        add rsp, 30h

        xor rcx, rcx
        call ExitProcess
    Main endp
end

Please help me. Thanks in advance for Your help.

Answer 1

you incorrect operate with stack. your main error - push rax really you setup here not 5-th argument but nothing. and 5-th argument was random value in stack. correct setup 5-th argument - mov [rsp+20h],rax . look more also on stack align, and GetLastError not related here - Reg* functions return error code yorself but not setup lasterror. and not use global variable for hReg, but allocate it in stack. examle can look like this:

extrn ExitProcess : proc
extrn MessageBoxA : proc
extrn RegOpenKeyExA : proc
extrn RegSetValueExA : proc
extrn RegCloseKey : proc

.const
HKEY_CURRENT_USER equ 80000001h
KEY_SET_VALUE equ 2
KEY_CREATE_SUB_KEY equ 4
KEY_WOW64_64KEY equ 0100h
KEY_WRITE equ 00020006h
REG_SZ equ 00000001h

.data
szRegKeyName db "Software\Microsoft\Windows\CurrentVersion\Run",0

.code

hReg equ 28h

    test proc
        sub rsp, 38h
        lea rax,[rsp+hReg]
        mov [rsp+20h],rax
        mov r9, KEY_SET_VALUE+KEY_CREATE_SUB_KEY+KEY_WOW64_64KEY
        mov r8, 0
        lea rdx, szRegKeyName
        mov rcx, HKEY_CURRENT_USER
        call RegOpenKeyExA
        test eax,eax
        jnz @@1
        mov rcx,[rsp+hReg]
        call RegCloseKey
@@1:
        add rsp, 38h
        ret
    test endp
end