Search code examples
c++visual-studioassemblyx86masm

Unexpected page fault when writing return value from assembly procedure


I am experimenting mixing assembly(x86) and C++. I wrote a procedure in assembly and then called it from C++.

However, when writing the returned value to a local variable I get a write permission violation error.

#include <iostream>


// will return 1 if all ok and 0 if b is 0
extern "C" int integerMulDiv(int a, int b, int* prod, int* quo, int* rem);

int main() {
    int a = 13, b = 4;
    int p, q, r;

    int res = integerMulDiv(a, b, &p, &q, &r);
    std::cout << p << '\t' << q << '\t' << r << std::endl;
    std::cout << res << std::endl << std::endl;

    res = integerMulDiv(31, 0, &p, &q, &r);
    std::cout << p << '\t' << q << '\t' << r << std::endl;
    std::cout << res << std::endl << std::endl;

    return 0;
}

The assembly procedure returns a few values through pointers and an int through RAX.

; Returns : 0 Error (division by 0)
;         : 1 All ok

; *prod = a * b
; *quo  = a / b
; *rem  = a % b
integerMulDiv proc

    push ebp
    mov ebp, esp
    push ebx  ; save ebp and ebx

    xor eax, eax

    mov ecx, [ebp + 8]  ; get a
    mov edx, [ebp + 12] ; get b (the divisor)

    or edx, edx ; check divisor
    jz invalidDivizor

    imul edx, ecx
    mov ebx, [ebp + 16] ; get address of prod
    mov [ebx], edx      ; write prod

    mov eax, ecx
    cdq ; extend to edx
    idiv dword ptr[ebx + 12]

    mov ebx, [ebp + 20] ; get address of quo
    mov [ebp], eax      ; write quo
    mov ebx, [ebp + 24] ; get address of rem
    mov [ebp], edx      ; write rem

    mov eax, 1          ; set success
    jmp returnFromProc

invalidDivizor:
    mov eax, 0          ; set failed

returnFromProc:
    pop ebx
    pop ebp
    ret   ; restore and return

integerMulDiv endp

I get the error after the first call of integerMulDiv, when it tries to write the result in the res variable.

The disassembly looks like this:

    int res = integerMulDiv(a, b, &p, &q, &r);
002D24BD  lea         eax,[r]  
002D24C0  push        eax  
002D24C1  lea         ecx,[q]  
002D24C4  push        ecx  
002D24C5  lea         edx,[p]  
002D24C8  push        edx  
002D24C9  mov         eax,dword ptr [b]  
002D24CC  push        eax  
002D24CD  mov         ecx,dword ptr [a]  
002D24D0  push        ecx  
002D24D1  call        _integerMulDiv (02D133Eh)  
002D24D6  add         esp,14h  
002D24D9  mov         dword ptr [res],eax   <- The #PF happens here

Does anyone know what is happening and why?


Solution

  • The following section of code stands out to me.

    idiv dword ptr[ebx + 12]
    
    mov ebx, [ebp + 20] ; get address of quo
    mov [ebp], eax      ; write quo
    mov ebx, [ebp + 24] ; get address of rem
    mov [ebp], edx      ; write rem
    

    I am not sure you are wanting to divide by the contents of memory 12 bytes after the address of the product. Perhaps you meant [ebp + 12].

    After that, you are loading addresses into ebx and then writing values to ebp.