I have a virtual machine, which on VM_Create passes the address of a function (systemCalls) to the virtual machine.
So I hook VM_Create and steal the syscalls address, put it into a backup function pointer, and the address of my modified systemCalls function pass to the original VM_Create, from which I can alter arguments, add or remove calls, and then call the backed-up syscalls function. That worked well, until a new release of the game.
I believe to have found the problem:
This is the beginning of the unmodified systemCalls function:
intptr_t CL_CgameSystemCalls(intptr_t *args) {
switch (args[0]) {
case CG_PRINT:
Com_Printf( "%s", (const char*)VMA(1));
return 0;
case CG_ERROR:
Com_Error(ERR_DROP, "%s", (const char*)VMA(1));
return 0;
This is my modifed syscall function:
intptr_t modified_CL_CgameSystemCalls (intptr_t *args)
{
switch (*args)
{
case CG_GETSNAPSHOT:
mysnap = mysnap ;
mynextSnap = (snapshot_t*) (CG_QVM2NATIVE(args[2]));
mysnap = mynextSnap;
retval = original_CL_CgameSystemCalls(args);
break ;
The problem is calling the original function from the modified one:
intptr_t modified_CL_CgameSystemCalls(intptr_t *args)
{
retval = original_CL_CgameSystemCalls(args);
return retval;
}
already fails.
As you can see from the pseudocode of the disassembly, the new definition of CL_CgameSystemCalls seems to be:
char __usercall sub_4017B0<al>(int a1<ebx>, int a2)
Which means they changed the function by adding the __usercall attribute and by putting the first argument into register ebx, if I interpret the decompilation right.
Now my question:
How can I retrieve *args (args[0]) into a variable?
And how can I call the unmodified function from the modified one, which now uses __usercall?
This is the disassembly of systemcalls with usercall:
.text:004017B0 ; =============== S U B R O U T I N E =======================================
.text:004017B0
.text:004017B0
.text:004017B0 sub_4017B0 proc near ; DATA XREF: sub_402670+5Co
.text:004017B0
.text:004017B0 var_18 = dword ptr -18h
.text:004017B0 var_4 = dword ptr -4
.text:004017B0 arg_0 = dword ptr 4
.text:004017B0
.text:004017B0 ; FUNCTION CHUNK AT .text:00401430 SIZE 00000026 BYTES
.text:004017B0 ; FUNCTION CHUNK AT .text:00401459 SIZE 00000013 BYTES
.text:004017B0 ; FUNCTION CHUNK AT .text:00410E90 SIZE 00000006 BYTES
.text:004017B0 ; FUNCTION CHUNK AT .text:00412AC0 SIZE 00000006 BYTES
.text:004017B0
.text:004017B0 push esi
.text:004017B1 mov esi, [esp+0Ch+var_4]
.text:004017B5 mov eax, [esi]
.text:004017B7 cmp eax, 73h ; switch 116 cases
.text:004017BA push edi
.text:004017BB ja loc_402486 ; default
.text:004017BB ; jumptable 004017C1 cases 21,90-99,109,110
.text:004017C1 jmp ds:off_40249C[eax*4] ; switch jump
.text:004017C8
.text:004017C8 loc_4017C8: ; DATA XREF: .text:off_40249Co
.text:004017C8 mov eax, [esi+4] ; jumptable 004017C1 case 0
.text:004017CB push eax
.text:004017CC call VM_ArgPtr
.text:004017D1 push eax ; char
.text:004017D2 push offset aS_5 ; "%s"
.text:004017D7 call Com_Printf
.text:004017DC add esp, 0Ch
.text:004017DF pop edi
.text:004017E0 xor eax, eax
.text:004017E2 pop esi
.text:004017E3 retn
.text:004017E4 ; ---------------------------------------------------------------------------
.text:004017E4
.text:004017E4 loc_4017E4: ; CODE XREF: sub_4017B0+11j
.text:004017E4 ; DATA XREF: .text:off_40249Co
.text:004017E4 mov ecx, [esi+4] ; jumptable 004017C1 case 1
.text:004017E7 push ecx
.text:004017E8 call VM_ArgPtr
.text:004017ED push eax ; char
.text:004017EE push offset aS_5 ; "%s"
.text:004017F3 push 1 ; int
.text:004017F5 call Com_Error
.text:004017F5 ; --------------------------------------------------------------
And this is the pseudocode created from the hexrays decompiler:
char __usercall sub_4017B0<al>(int a1<ebx>, int a2)
{
int v2; // ST34_4@1
char result; // al@2
int v4; // ST34_4@2
int v5; // eax@2
int v6; // ST34_4@3
int v7; // eax@3
int v8; // ST34_4@5
int v9; // ST24_4@5
int v10; // ST20_4@5
int v11; // ST1C_4@5
int v12; // eax@5
int v13; // ST34_4@6
int v14; // eax@6
int v15; // ST34_4@7
int v16; // ST24_4@7
int v17; // eax@7
int v18; // ST34_4@8
signed int v19; // ST24_4@8
int v20; // ST20_4@8
int v21; // eax@8
int v22; // ST34_4@10
signed int v23; // ST24_4@10
int v24; // eax@10
int v25; // ST34_4@11
signed int v26; // ST24_4@11
int v27; // eax@11
int v28; // ST34_4@12
int v29; // ST24_4@12
int v30; // ST20_4@12
int v31; // eax@12
int v32; // ST34_4@13
int v33; // ST24_4@13
int v34; // ST20_4@13
int v35; // eax@13
int v36; // ST34_4@14
int v37; // ST24_4@14
size_t v38; // ST20_4@14
int v39; // eax@14
int v40; // ST34_4@15
int v41; // ST34_4@16
int v42; // ST34_4@17
int v43; // eax@17
int v44; // ST34_4@18
int v45; // eax@18
int v46; // ST34_4@19
int v47; // eax@19
int v48; // ST34_4@20
int v49; // eax@20
int v50; // ST34_4@22
int v51; // eax@22
void *v52; // ecx@22
int v53; // ST34_4@24
int v54; // ST20_4@24
int v55; // eax@24
int v56; // ST34_4@25
int v57; // ST20_4@25
int v58; // eax@25
int v59; // ST34_4@26
signed int v60; // ST24_4@26
int v61; // eax@26
int v62; // ST34_4@27
int v63; // ST24_4@27
int v64; // ST20_4@27
signed int v65; // ST1C_4@27
int v66; // eax@27
int v67; // ST34_4@28
int v68; // eax@29
int v69; // ST34_4@30
int v70; // ST34_4@32
int v71; // ST20_4@32
int v72; // ST1C_4@32
int v73; // ST18_4@32
signed int v74; // ST14_4@32
int v75; // ST10_4@32
int v76; // ST0C_4@32
int v77; // ST08_4@32
int v78; // ST04_4@32
int v79; // eax@32
int v80; // ST34_4@34
int v81; // ST24_4@34
int v82; // ST20_4@34
int v83; // ST1C_4@34
int v84; // ST18_4@34
int v85; // ST14_4@34
int v86; // eax@34
int v87; // ST34_4@35
int v88; // ST24_4@35
int v89; // ST20_4@35
int v90; // ST1C_4@35
int v91; // eax@35
int v92; // ST34_4@36
int v93; // ST34_4@37
int v94; // ST34_4@38
int v95; // ST24_4@38
int v96; // ST20_4@38
int v97; // eax@38
int v98; // ST34_4@39
int v99; // ST24_4@39
int v100; // ST20_4@39
int v101; // eax@39
int v102; // ST34_4@40
int v103; // ST34_4@41
int v104; // eax@41
int v105; // ST34_4@42
int v106; // ST20_4@42
int v107; // eax@42
int v108; // ST34_4@43
int v109; // eax@43
int v110; // ST34_4@44
int v111; // ST24_4@44
int v112; // eax@44
int v113; // ST34_4@45
int v114; // eax@45
int v115; // ST34_4@46
int v116; // eax@46
int v117; // ST34_4@47
int v118; // ST34_4@48
int v119; // ST34_4@49
int v120; // ST34_4@50
int v121; // ST24_4@50
int v122; // ST20_4@50
int v123; // eax@50
int v124; // ST34_4@52
int v125; // eax@52
int v126; // ST34_4@53
int v127; // eax@53
int v128; // ST34_4@54
int v129; // ST24_4@54
int v130; // eax@54
int v131; // ST34_4@55
int v132; // ST24_4@55
int v133; // ST20_4@55
int v134; // ST1C_4@55
int v135; // eax@55
int v136; // ST34_4@56
float v137; // ST24_4@56
float v138; // ST20_4@56
float v139; // ST1C_4@56
float v140; // ST18_4@56
int v141; // eax@56
int v142; // ST34_4@57
float v143; // ST24_4@57
float v144; // ST20_4@57
float v145; // ST1C_4@57
float v146; // ST18_4@57
int v147; // eax@57
int v148; // ST34_4@58
int v149; // eax@58
int v150; // ST34_4@59
int v151; // eax@59
int v152; // ST34_4@60
int v153; // ST34_4@61
int v154; // ST24_4@61
int v155; // eax@61
int v156; // ST34_4@62
int v157; // ST24_4@62
float v158; // ST20_4@62
int v159; // ST1C_4@62
int v160; // ST18_4@62
int v161; // ST14_4@62
int v162; // eax@62
int v163; // ST34_4@63
int v164; // eax@63
int v165; // ST34_4@64
int v166; // eax@64
int v167; // ST34_4@65
int v168; // edi@65
int v169; // ST34_4@66
int v170; // eax@66
int v171; // ST34_4@68
int v172; // eax@68
int v173; // edx@69
int v174; // ST34_4@69
float v175; // ST34_4@69
int v176; // ST34_4@72
int v177; // ST34_4@73
int v178; // ST34_4@74
unsigned int v179;// ST24_4@74
int v180; // ST20_4@74
int v181; // eax@74
int v182; // ST34_4@75
signed int v183; // ST24_4@75
int v184; // ST20_4@75
int v185; // eax@75
int v186; // ST34_4@76
size_t v187; // ST24_4@76
int v188; // ST20_4@76
int v189; // eax@76
int v190; // ST34_4@77
float v191; // ST34_4@77
int v192; // ST34_4@78
float v193; // ST34_4@78
double v194; // st7@79
int v195; // ST34_4@79
float v196; // ST34_4@79
int v197; // ST34_4@80
float v198; // ST34_4@80
int v199; // ST34_4@81
float v200; // ST34_4@81
int v201; // ST34_4@82
float v202; // ST34_4@82
int v203; // ST34_4@83
float v204; // ST34_4@83
int v205; // ST34_4@84
int v206; // edi@84
int v207; // eax@84
int v208; // ST34_4@85
int v209; // edi@85
int v210; // eax@85
int v211; // ST34_4@87
int v212; // edi@87
int v213; // eax@87
int v214; // ST34_4@88
int v215; // edi@88
int v216; // ST24_4@88
int v217; // eax@88
int v218; // ST34_4@90
int v219; // ST34_4@91
int v220; // eax@91
int v221; // ST34_4@92
unsigned int v222;// ST24_4@92
int v223; // ST20_4@92
int v224; // ST1C_4@92
int v225; // ST18_4@92
int v226; // ST14_4@92
int v227; // eax@92
int v228; // ST34_4@95
int v229; // ST34_4@96
int v230; // ST34_4@97
int v231; // ST24_4@97
int v232; // ST20_4@97
int v233; // eax@97
int v234; // ST34_4@98
int v235; // ST24_4@98
int v236; // eax@98
int v237; // ST34_4@99
int v238; // ST24_4@99
int v239; // eax@99
int v240; // ST34_4@100
int v241; // eax@100
int v242; // ST34_4@101
int v243; // eax@101
int v244; // ST34_4@102
int v245; // eax@102
int v246; // ST34_4@103
int v247; // ST20_4@103
int v248; // eax@103
int v249; // ST34_4@104
int v250; // [sp-18h] [bp-30h]@28
int v251; // [sp-14h] [bp-2Ch]@28
int v252; // [sp-10h] [bp-28h]@28
int v253; // [sp-Ch] [bp-24h]@28
int v254; // [sp-8h] [bp-20h]@28
signed int v255; // [sp-4h] [bp-1Ch]@28
int v256; // [sp+0h] [bp-18h]@28
int v257; // [sp+4h] [bp-14h]@28
int v258; // [sp+4h] [bp-14h]@31
switch (*(_DWORD *)v2)
{
case 0:
v5 = VM_ArgPtr(*(_DWORD *)(v4 + 4));
Com_Printf("%s", v5);
return 0;
case 1:
v7 = VM_ArgPtr(*(_DWORD *)(v6 + 4));
Com_Error(1, "%s", v7);
return result;
case 2:
return sub_447700();
case 3:
v9 = *(_DWORD *)(v8 + 16);
v10 = VM_ArgPtr(*(_DWORD *)(v8 + 12));
v11 = VM_ArgPtr(*(_DWORD *)(v8 + 8));
v12 = VM_ArgPtr(*(_DWORD *)(v8 + 4));
sub_4213C0(v12, (const char *)v11, v10, v9);
return 0;
case 4:
...
default:
Com_Error(1, "Bad cgame system trap: %i", *(_DWORD *)v249);
return result;
}
return result;
}
You can find the full (last known official) source of the CL_CgameSystemCalls function here (too much to copy-paste):
http://ioqsrc.vampireducks.com/da/d3b/cl__cgame_8c-source.html
And this here is the disassembly of the old version, in which calling orig_syscall from modified syscall worked:
.text:00402B40 ; =============== S U B R O U T I N E =======================================
.text:00402B40
.text:00402B40
.text:00402B40 sub_402B40 proc near ; CODE XREF: sub_40B380+Bp
.text:00402B40 ; sub_40E3B0+Bp ...
.text:00402B40 mov edx, dword_BBE104
.text:00402B46 mov eax, dword_CB60EC
.text:00402B4B and edx, 0FFFFFFF7h
.text:00402B4E test eax, eax
.text:00402B50 mov dword_BBE104, edx
.text:00402B56 mov dword_BBE21C, 0
.text:00402B60 jz short locret_402B82
.text:00402B62 push 1
.text:00402B64 push eax
.text:00402B65 call sub_43E360
.text:00402B6A mov eax, dword_CB60EC
.text:00402B6F push eax
.text:00402B70 call sub_43E270
.text:00402B75 add esp, 0Ch
.text:00402B78 mov dword_CB60EC, 0
.text:00402B82
.text:00402B82 locret_402B82: ; CODE XREF: sub_402B40+20j
.text:00402B82 retn
.text:00402B82 sub_402B40 endp
.text:00402B82
.text:00402B82 ; ---------------------------------------------------------------------------
.text:00402B83 align 10h
.text:00402B90
.text:00402B90 loc_402B90: ; DATA XREF: sub_403AA0+5Co
.text:00402B90 push ecx
.text:00402B91 push ebx
.text:00402B92 push esi
.text:00402B93 push edi
.text:00402B94 mov edi, [esp+14h]
.text:00402B98 mov eax, [edi]
.text:00402B9A cmp eax, 6Fh ; switch 112 cases
.text:00402B9D ja loc_4038C7 ; default
.text:00402B9D ; jumptable 00402BA3 cases 21,90-99,109,110
.text:00402BA3 jmp ds:off_4038E0[eax*4] ; switch jump
.text:00402BAA
.text:00402BAA loc_402BAA: ; DATA XREF: .text:off_4038E0o
.text:00402BAA mov eax, [edi+4] ; jumptable 00402BA3 case 0
.text:00402BAD push eax
.text:00402BAE call sub_43E300
.text:00402BB3 push eax
.text:00402BB4 push offset aS_7 ; "%s"
.text:00402BB9 call sub_41BB90
.text:00402BBE add esp, 0Ch
.text:00402BC1 pop edi
.text:00402BC2 pop esi
.text:00402BC3 xor eax, eax
.text:00402BC5 pop ebx
.text:00402BC6 pop ecx
.text:00402BC7 retn
.text:00402BC8 ; ---------------------------------------------------------------------------
.text:00402BC8
.text:00402BC8 loc_402BC8: ; CODE XREF: .text:00402BA3j
.text:00402BC8 ; DATA XREF: .text:off_4038E0o
.text:00402BC8 mov ecx, [edi+4] ; jumptable 00402BA3 case 1
.text:00402BCB push ecx
.text:00402BCC call sub_43E300
.text:00402BD1 push eax
.text:00402BD2 push offset aS_7 ; "%s"
.text:00402BD7 push 1
.text:00402BD9 call sub_41D850
.text:00402BDE ; ---------------------------------------------------------------------------
There is no such thing as __usercall. Hex-rays uses this to represent a non-standard calling convention. This can happen in three ways:
#pragma aux to define a custom conventionIn cases like this, there are only two ways to go, use assembly or use Open Watcom (which just got an update after years of abandonment). Inline assembly works best IMO, but you'll need a wrapper going in (to call the original) and a wrapper coming out (to hook the original).