Return value of mmap when using ptrace

Question

I was learning how to use ptrace and I faced a strange problem:

I wrote a program:

#include <cstdio>
#include <sys/mman.h>
#include <string.h>
#include <errno.h>

int main()
{
    long x=(long)mmap(0,-235,2,34,-1,0);
    printf("Child: x=%ld (",x);
    for(int i=31;i>=0;i--) printf((x&(1<<i))?"1":"0");
    printf(")\n");
    printf("Child errno: %s\n",strerror(errno));
    return 0;
}

It simply makes an mmap syscall with wrong parameter. Then it prints return value (also in binary) and errno.

Here I have this program's output after executing it:

Child: x=-1 (11111111111111111111111111111111)
Child errno: Cannot allocate memory

And I run it with strace:

execve("./nic.e", ["./nic.e"], [/* 35 vars */]) = 0
uname({sys="Linux", node="dom", ...})   = 0
brk(0)                                  = 0x9237000
brk(0x9237cd0)                          = 0x9237cd0
set_thread_area({entry_number:-1 -> 6, base_addr:0x9237830, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
brk(0x9258cd0)                          = 0x9258cd0
brk(0x9259000)                          = 0x9259000
mmap2(NULL, 4294967061, PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = -1 ENOMEM (Cannot allocate memory)
fstat64(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7798000
write(1, "Child: x=-1 (1111111111111111111"..., 47Child: x=-1 (11111111111111111111111111111111)
) = 47
write(1, "Child errno: Cannot allocate mem"..., 36Child errno: Cannot allocate memory
) = 36
exit_group(0)                           = ?

And strace tells that this wrong mmap returns -1 with error ENOMEM.

Till now everything is OK.

Here my code with ptrace (I cut everything not really needed):

#include <cstdio>
#include <cstdlib>
#include <unistd.h>
#include <sys/resource.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/ptrace.h>
#include <sys/user.h>
#include <sys/wait.h>
#include <sys/syscall.h>
#include <sys/reg.h>
#include <errno.h>
#include <string.h>
#include <fcntl.h>

int main(int argc,char**argv)
{
    int pid=fork();
    if(!pid)
    {
        ptrace(PTRACE_TRACEME,0,NULL,NULL);
        execve("nic.e",NULL,NULL);
        exit(1);
    }

    while(true)
    {
        int status;
        waitpid(pid,&status,0);

        if(WIFEXITED(status)) return 0;

        int signal;

        if(WIFSTOPPED(status))
        {
            signal=WSTOPSIG(status);
        }

        if(WIFSIGNALED(status)) return 0;

        if(signal==SIGTRAP)
        {
            user_regs_struct regs;
            ptrace(PTRACE_GETREGS,pid,NULL,&regs);

            if(regs.orig_eax==__NR_mmap2)
            {
                static bool mmap_back=false;
                if(!mmap_back) mmap_back=true;
                else
                {
                    mmap_back=false;
                    long x=regs.eax;
                    printf("mmap return: %ld (",x);
                    for(int j=31;j>=0;j--) printf((x&(1<<j))?"1":"0");
                    printf(")\n");
                }
            }
        }
        ptrace(PTRACE_SYSCALL,pid,NULL,NULL);
    }
    return 0;
}

It should print same things that the child prints - retrun values of mmap2 syscalls.

But here's the output:

mmap return: -12 (11111111111111111111111111110100)
mmap return: -1216753664 (10110111011110011101000000000000)
Child: x=-1 (11111111111111111111111111111111)
Child errno: Cannot allocate memory

Why did mmap return -12? Am I capturing the return value incorrectly?

Answer 1

On 32-bit x86 linux %eax contains either the return value or the negated error value on error.

See e.g. 4.4 or 3.3 and 3.4.

A return value of -12 from the syscall means that the function failed and errno should be set to 12, which at least on my system matches ENOMEM.

It appears that strace is helpfully translating this for you. If you want your application to behave as strace, you should perform a test and translation similar to the one in 4.4:

if ((unsigned long)(x) >= (unsigned long)(-2048)) {
    printf("syscall failed. errno = %ld\n", -(res));
}