I'm trying to recreate using MASM /w Visual Studio 2019 something similar to the following C++ code which works. Essentially at this stage just want the window to be movable and the close button to work.
#include <iostream>
#include <Windows.h>
LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM
lParam)
{
switch (message)
{
case WM_DESTROY:
PostQuitMessage(0);
break;
default:
return DefWindowProc(hWnd, message, wParam, lParam);
}
return 0;
}
int main()
{
wchar_t windowclass[] = L"MyWinTest";
HINSTANCE hInstance = GetModuleHandleW(NULL);
MSG msg;
WNDCLASSEXW wc;
wc.cbSize = sizeof(WNDCLASSEX);
wc.cbClsExtra = 0;
wc.cbWndExtra = 0;
wc.hbrBackground = 0;
wc.hCursor = LoadCursor(NULL, IDC_ARROW);
wc.hIcon = LoadIcon(hInstance, IDI_APPLICATION);
wc.hIconSm = NULL;
wc.hInstance = hInstance;
wc.lpfnWndProc = WndProc;
wc.lpszClassName = windowclass;
wc.lpszMenuName = nullptr;
wc.style = CS_HREDRAW | CS_VREDRAW;
RegisterClassExW(&wc);
HWND hWnd = CreateWindowExW(
0,
windowclass,
L"MyWindow",
WS_OVERLAPPEDWINDOW,
CW_USEDEFAULT,
CW_USEDEFAULT,
800,
600,
NULL,
NULL,
hInstance,
NULL);
ShowWindow(hWnd, SW_SHOW);
UpdateWindow(hWnd);
while (GetMessage(&msg, NULL, 0, 0))
{
TranslateMessage(&msg);
DispatchMessage(&msg);
}
}
In my version of x64 assembly using MASM the Window is created and displayed, also WndProc is being hit, can see it get message WM_CREATE and WM_PAINT but the Window can't be moved and can't be closed. I did compile the C++ version as x64 and output assembly listing for comparison, but it looks very similar as far as I can tell to the assembly version.
WSTR MACRO lbl:req,qstr:VARARG
LOCAL arg,unq,qot,q
lbl LABEL WORD
FOR arg,<qstr>
qot SubStr <arg>,1,1
q = 0
IFIDNI qot,<!'>;'
q = 1
ELSEIFIDNI qot,<!">;"
q = 1
ELSE
DW arg
ENDIF
IF q EQ 1
unq SubStr <arg>,2,@SizeStr(<arg>)-2
% FORC c,<unq>
DW "&c"
ENDM
ENDIF
ENDM
DW 0
ENDM
L MACRO qstr:VARARG
LOCAL sym,seg
seg EQU <.code>
%IFIDNI <@CurSeg>,<_DATA>
seg EQU <.data>
ENDIF
.CONST
ALIGN 4
WSTR sym,qstr
seg
EXITM <OFFSET sym>
ENDM
extrn LoadCursorW: PROC
extrn MessageBoxW: PROC
extrn ExitProcess: PROC
extrn GetModuleHandleW: PROC
extrn RegisterClassExW: PROC
extrn CreateWindowExW: PROC
extrn GetLastError: PROC
extrn DefWindowProcW: PROC
extrn ShowWindow: PROC
extrn Sleep: PROC
extrn GetMessageW: PROC
extrn TranslateMessage: PROC
extrn DispatchMessageW: PROC
extrn DestroyWindow: PROC
extrn UpdateWindow: PROC
extrn PostQuitMessage: PROC
extrn BeginPaint: PROC
extrn EndPaint: PROC
.data
wstr windowClassName,"AsmTestClass",0,0
wstr windowTitle,"AsmTest",0,0
HWND_DESKTOP textequ <0h>
MB_OK textequ <0h>
INFINITE textequ <0ffffffffh>
WM_CREATE textequ <0001h>
WM_DESTROY textequ <0002h>
WM_SIZE textequ <0005h>
WM_PAINT textequ <000fh>
WM_CLOSE textequ <0010h>
WM_QUIT textequ <0012h>
SW_HIDE textequ <0000h>
SW_SHOW textequ <0005h>
CS_VREDRAW textequ <0001h>
CS_HREDRAW textequ <0002h>
WS_OVERLAPPED textequ <00000000h>
WS_CAPTION textequ <00c00000h>
WS_SYSMENU textequ <00080000h>
WS_MINIMIZEBOX textequ <00020000h>
CW_USEDEFAULT textequ <80000000h>
IDI_APPLICATION textequ <00007f00h>
WINDOW_WIDTH DWORD 800
WINDOW_HEIGHT DWORD 600
WNDCLASSEX STRUCT DWORD
cbSize DWORD ?
style DWORD ?
lpfnWndProc QWORD ?
cbClsExtra DWORD ?
cbWndExtra DWORD ?
hInstance QWORD ?
hIcon QWORD ?
hCursor QWORD ?
hbrBackground QWORD ?
lpszMenuName QWORD ?
lpszClassName QWORD ?
hIconSm QWORD ?
WNDCLASSEX ENDS
MSG STRUCT
hwnd QWORD ?
message DWORD ?
wParam QWORD ?
lParam QWORD ?
time DWORD ?
x DWORD ?
y DWORD ?
MSG ENDS
PAINTSTRUCT STRUCT 8
hdc QWORD ?
fErase DWORD ?
left DWORD ?
top DWORD ?
right DWORD ?
bottom DWORD ?
fRestore DWORD ?
fIncUpdate DWORD ?
rgbReserved BYTE 32 DUP (?)
PAINTSTRUCT ENDS
.code
main proc
LOCAL wc:WNDCLASSEX
LOCAL hWnd:QWORD
LOCAL hInstance:QWORD
LOCAL hCursor:QWORD
LOCAL ATOM:WORD
LOCAL message:MSG
sub rsp, 8
; hInstance = GetModuleHandle(NULL)
mov rcx, 0
call GetModuleHandleW
mov hInstance, rax
; hCursor = LoadCursor(NULL,IDI_APPLICATION)
mov edx, IDI_APPLICATION
xor ecx, ecx
call LoadCursorW
mov hCursor, rax
; Setup Window Class
mov wc.cbSize, SIZEOF WNDCLASSEX
mov wc.style, CS_VREDRAW or CS_HREDRAW
lea rax, OFFSET WndProc
mov wc.lpfnWndProc, rax
mov wc.cbClsExtra, 0
mov wc.cbWndExtra, 0
lea rax, hInstance
mov wc.hInstance, rax
mov wc.hbrBackground, 0
mov wc.lpszMenuName, 0
lea rax, hCursor
mov wc.hCursor, rax
lea rax, windowClassName
mov wc.lpszClassName, rax
mov wc.hIconSm, 0
lea rcx, wc
call RegisterClassExW
mov ATOM, ax
; CreateWindowExW
mov QWORD PTR [rsp+88], 0 ; lpParam
lea rax, hInstance
mov QWORD PTR [rsp+80], rax ; hInstance
mov QWORD PTR [rsp+72], 0 ; hMenu
mov QWORD PTR [rsp+64], 0 ; hWndParent
mov edx, WINDOW_HEIGHT
mov DWORD PTR [rsp+56], edx ; nHeight
mov edx, WINDOW_WIDTH
mov DWORD PTR [rsp+48], edx ; nWidth
mov DWORD PTR [rsp+40], CW_USEDEFAULT ; Y
mov DWORD PTR [rsp+32], CW_USEDEFAULT ; X
mov r9d, WS_OVERLAPPED or WS_CAPTION or WS_SYSMENU or WS_MINIMIZEBOX ; dwStyle
lea r8, windowTitle ; lpWindowName
lea rdx, windowClassName ; lpClassName
xor ecx,ecx ; dwExStyle
call CreateWindowExW
cmp rax, 0
je WindowFailed
jmp WindowSuccess
WindowFailed:
call GetLastError
; to-do check error ?
WindowSuccess:
mov hWnd, rax
mov rcx, hWnd ; hWnd
mov edx, SW_SHOW ; nCmdShow
call ShowWindow
mov rcx, hWnd ; hWnd
call UpdateWindow
MessageLoop:
xor r9d, r9d ; wMsgFilterMax
xor r8d, r8d ; wMsgFilterMin
xor edx, edx ; hWnd
lea rcx, message ; lpMsg
call GetMessageW
test eax, eax
je QuitMessageLoop
lea rcx, message ; lpMsg
call TranslateMessage
lea rcx, message ; lpMsg
call DispatchMessageW
jmp MessageLoop
QuitMessageLoop:
mov ecx, eax ; uExitCode
call ExitProcess
main endp
WndProc proc
LOCAL hWnd:QWORD
LOCAL uMsg:DWORD
LOCAL wParam:QWORD
LOCAL lParam:QWORD
LOCAL result:QWORD
LOCAL ps:PAINTSTRUCT
LOCAL hdc:QWORD
sub rsp, 8
mov lParam, r9
mov wParam, r8
mov uMsg, edx
mov hWnd, rcx
; msg handler
cmp uMsg,WM_CREATE
je create
cmp uMsg,WM_PAINT
je paint
cmp uMsg,WM_DESTROY
je destroy
; default
mov r9, lParam
mov r8, wParam
mov edx, uMsg
mov rcx, hWnd
call DefWindowProcW
mov result,rax
jmp finish
create:
mov result, 0
jmp finish
paint:
mov rcx, hWnd
lea rdx, ps
call BeginPaint
mov hdc, rax
; to-do HDC paint stuff here
mov rcx, hWnd
lea rdx, ps
call EndPaint
mov result, 0
jmp finish
destroy:
xor ecx, ecx ; nExitCode
call PostQuitMessage
mov result, 0
jmp finish
finish:
mov rax, result
ret
WndProc endp
End
Note due to local statement masm is automatically adding to my main before the sub rsp,8: (-152 + -8 = -160 )
push rbp
mov rbp, rsp
add rsp, 0FFFFFFFFFFFFFF68h
and adding to wndproc routine (-120 + -8 = -128)
push rbp
mov rbp, rsp
add rsp, 0FFFFFFFFFFFFFF88h
...
leave
error in code - in both procs -
sub rsp, 8
when from x64 calling convention
The caller is responsible for allocating space for the callee's parameters. The caller must always allocate sufficient space to store four register parameters, even if the callee doesn't take that many parameters.
so need - to store four register parameters - 4*8 and +8 for 16-byte align, so
sub rsp, 40
in concrete case calls to DispatchMessageW and TranslateMessage can corrupt (overwrite) message because it located in callee's parameters space. if test under debugger - i view that DispatchMessageW overwrite message in prolog and than already use wrong message