DS_CONTROL | WS_CHILD combination causes infinite loop

Question

Hello.

I'm trying to create a "dialog in dialog" example using the WinAPI and C. This example consists in a child dialog with an autocheckbox, and a main dialog containing a static black rectangle which is the parent for the child dialog and a push button that shows in a message box a text with the checkbox status.

When I set the flags DS_CONTROL | WS_CHILD for the child dialog, whenever I try to change the checkbox status the application enters into an infinite loop and I have to force close it. When I remove the DS_CONTROL flag, works as intended but I cannot cycle between the controls using the tab key.

What can I do in order to make it work as intended using the DS_CONTROL flag?

Here's the content of my main.c file:

#include <windows.h>

#pragma comment (lib, "user32")


HINSTANCE hInst;
BOOL isChecked;
const unsigned char checkedStr[] = "Checkbox is checked";
const unsigned char notCheckedStr[] = "Checkbox is not checked";


BOOL CALLBACK ChildDlgProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
  switch (uMsg)
  {
    case WM_COMMAND:
      switch (LOWORD(wParam))
      {
        case 21:
          isChecked = IsDlgButtonChecked(hwndDlg, 21);
          return TRUE;
      }

      return FALSE;
  }

  return FALSE;
}


BOOL CALLBACK DialogProc(HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
  switch (uMsg)
  {
    case WM_INITDIALOG:
    {
      HWND hContainer, hChilddDlg;

      hContainer = GetDlgItem(hwndDlg, 11);
      hChilddDlg = CreateDialogParam(hInst, MAKEINTRESOURCE(20), hContainer, ChildDlgProc, 0);
      ShowWindow(hChilddDlg, SW_SHOW);

      return TRUE;
    }

    case WM_COMMAND:
      switch (LOWORD(wParam))
      {
        case 12:
        {
          const unsigned char *ptr;

          if (isChecked)
          {
            ptr = checkedStr;
          }
          else
          {
            ptr = notCheckedStr;
          }

          MessageBox(hwndDlg, ptr, TEXT("Checkbox status"), MB_OK | MB_ICONINFORMATION);

          return TRUE;
        }
      }

      return FALSE;

    case WM_CLOSE:
      EndDialog(hwndDlg, 0);
      return TRUE;
  }


  return FALSE;
}


int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
{
  hInst = hInstance;
  isChecked = FALSE;

  return DialogBoxParam(hInstance, MAKEINTRESOURCE(10), NULL, DialogProc, 0);
}

And here's the content of my rsrc.rcfile:

#include <windows.h>

10 DIALOGEX 0, 0, 130, 47
STYLE DS_CENTER | DS_SETFONT | WS_CAPTION | WS_MINIMIZEBOX | WS_SYSMENU
CAPTION "Checkbox status"
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
FONT 9, "Segoe UI"
{
  CONTROL "", 11, STATIC, SS_BLACKRECT | WS_CHILD | WS_VISIBLE, 5, 5, 120, 20
  CONTROL "&Status", 12, BUTTON, BS_DEFPUSHBUTTON | WS_CHILD | WS_VISIBLE | WS_TABSTOP, 75, 30, 50, 12
}


20 DIALOGEX 0, 0, 120, 20
STYLE DS_SETFONT | DS_CONTROL | WS_CHILD
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
FONT 9, "Segoe UI"
{
  CONTROL "Checkbox", 21, BUTTON, BS_AUTOCHECKBOX | WS_CHILD | WS_VISIBLE | WS_TABSTOP, 0, 5, 50, 10
}

I compile it in the Visual C++ command prompt with: cl /c main.c && rc rsrc.rc && link /SUBSYSTEM:WINDOWS /OUT:test.exe main.obj rsrc.res.

Thanks in advance for your help.

Answer 1

Found the solution in the NSIS source code. The problem was that I was putting the child dialog as a child of the blackrect so it was out of the event loop of the main dialog, causing the hang. In order to solve this I had to put it as a child of the main dialog and move it over the blackrect.

Here's the updated code of the WM_INITDIALOG case in the main dialog proc:

// ...
case WM_INITDIALOG:
{
  HWND hChilddDlg;

  hChilddDlg = CreateDialogParam(hInst, MAKEINTRESOURCE(20), hwndDlg, ChildDlgProc, 0);

  if (hChilddDlg)
  {
    RECT rect;

    GetWindowRect(GetDlgItem(hwndDlg, 11), &rect);
    ScreenToClient(hwndDlg, (LPPOINT)&rect);
    SetWindowPos(hChilddDlg, 0, rect.left, rect.top, 0, 0, SWP_NOACTIVATE | SWP_NOSIZE | SWP_NOZORDER);
    ShowWindow(hChilddDlg, SW_SHOWNA);
  }

  return TRUE;
}
// ...

With this I was able to use the DS_CONTROL flag and tab cycle between controls.