Search code examples
c++winapitransparencygdi+gdi

C++ Win32 API GDI : Rectangle AntiAliasing not working properly with transparent background


I use extended frame to render a custom caption and window border.

hr = DwmExtendFrameIntoClientArea(hWnd, &margins);

I also use layered window to render background transparent. My COLORKEY is RGB(0,0,0)

SetLayeredWindowAttributes(hWnd, RGB(0, 0, 0), 255, LWA_COLORKEY);

The reason I use layered window is, I want to make window's bottom border's corners rounded.

The issue is I want to do something beatiful and I tried to render a Anti Aliased window border in client area with GDI. However, It work as Anti Aliased with painted background (Solid) but not with TRANSPARENT background.

IMAGE: https://ibb.co/fD9GsFg

What should I do to fix this ?

If you try it please use VS debugger as I did not put functional window buttons.

#include <windows.h>
#include <tchar.h>
#include <windowsx.h>
#include <objidl.h>
#include <gdiplus.h>

#include <dwmapi.h>
#pragma comment(lib,"Dwmapi")

using namespace Gdiplus;
#pragma comment (lib,"Gdiplus.lib")

HINSTANCE hInst;
LRESULT CALLBACK WndProc(HWND, UINT, WPARAM, LPARAM);
int nWidth = 600, nHeight = 400;

#define RECTWIDTH(rc)            (rc.right - rc.left)
#define RECTHEIGHT(rc)            (rc.bottom - rc.top)

//CHANGE THEM TO 0 when thats maximized!
const int TOPEXTENDWIDTH = 48;
const int LEFTEXTENDWIDTH = 8;
const int RIGHTEXTENDWIDTH = 8;
const int BOTTOMEXTENDWIDTH = 8;

HBRUSH RED_BRUSH = CreateSolidBrush(RGB(237, 28, 36));
HBRUSH DARKBLUE_BRUSH = CreateSolidBrush(RGB(26, 31, 96));
HBRUSH PURPLE_BRUSH = CreateSolidBrush(RGB(163, 73, 164));
HBRUSH LIGHTPURPLE_BRUSH_1 = CreateSolidBrush(RGB(189, 106, 189));
HBRUSH LIGHTPURPLE_BRUSH_2 = CreateSolidBrush(RGB(255, 174, 201));
HBRUSH DARKEST_BRUSH = CreateSolidBrush(RGB(0, 0, 0));

LRESULT HitTestNCA(HWND hWnd, WPARAM wParam, LPARAM lParam);

int WINAPI wWinMain( _In_opt_ HINSTANCE hInstance, _In_opt_ HINSTANCE hPrevInstance, _In_opt_ LPTSTR lpCmdLine, _In_opt_ int nCmdShow)
{
    GdiplusStartupInput gdiplusStartupInput;
    ULONG_PTR           gdiplusToken;

    // Initialize GDI+.
    GdiplusStartup(&gdiplusToken, &gdiplusStartupInput, NULL);

    hInst = hInstance;
    WNDCLASSEX wcex =
    {
        sizeof(WNDCLASSEX), CS_HREDRAW | CS_VREDRAW, WndProc, 0, 0, hInst, LoadIcon(NULL, IDI_APPLICATION),
        LoadCursor(NULL, IDC_ARROW), (HBRUSH)GetStockObject(BLACK_BRUSH), NULL, TEXT("WindowClass"), NULL,
    };
    if (!RegisterClassEx(&wcex))
        return MessageBox(NULL, L"Cannot register class !", L"Error", MB_ICONERROR | MB_OK);
    int nX = (GetSystemMetrics(SM_CXSCREEN) - nWidth) / 2, nY = (GetSystemMetrics(SM_CYSCREEN) - nHeight) / 2;
    HWND hWnd = CreateWindowEx(0, wcex.lpszClassName, TEXT("Test"), WS_OVERLAPPEDWINDOW, nX, nY, nWidth, nHeight, NULL, NULL, hInst, NULL);
    
    if (!hWnd) return MessageBox(NULL, L"Cannot create window !", L"Error", MB_ICONERROR | MB_OK);

    //NO SHADOW
    SystemParametersInfoA(SPI_SETDROPSHADOW,0,(const PVOID) false,SPIF_SENDWININICHANGE);

    ShowWindow(hWnd, SW_SHOWNORMAL);
    UpdateWindow(hWnd);
    MSG msg;
    while (GetMessage(&msg, NULL, 0, 0))
    {
        TranslateMessage(&msg);
        DispatchMessage(&msg);
    }
    return (int)msg.wParam;
}
void FillRoundRectangle(Gdiplus::Graphics* g, Brush* p, Gdiplus::Rect& rect, UINT8 radius[4])
{
    if (g == NULL) return;
    GraphicsPath path;
    //TOP RIGHT
    path.AddLine(rect.X + radius[0], rect.Y, rect.X + rect.Width - (radius[0] * 2), rect.Y);
    path.AddArc(rect.X + rect.Width - (radius[0] * 2), rect.Y, radius[0] * 2, radius[0] * 2, 270, 90);

    //BOTTOM RIGHT
    path.AddLine(rect.X + rect.Width, rect.Y + radius[1], rect.X + rect.Width, rect.Y + rect.Height - (radius[1] * 2));
    path.AddArc(rect.X + rect.Width - (radius[1] * 2), rect.Y + rect.Height - (radius[1] * 2), radius[1] * 2, radius[1] * 2, 0, 90);

    //BOTTOM LEFT
    path.AddLine(rect.X + rect.Width - (radius[2] * 2), rect.Y + rect.Height, rect.X + radius[2], rect.Y + rect.Height);
    path.AddArc(rect.X, rect.Y + rect.Height - (radius[2] * 2), radius[2] * 2, radius[2] * 2, 90, 90);

    //TOP LEFT
    path.AddLine(rect.X, rect.Y + rect.Height - (radius[3] * 2), rect.X, rect.Y + radius[3]);
    path.AddArc(rect.X, rect.Y, radius[3] * 2, radius[3] * 2, 180, 90);
    path.CloseFigure();

    g->FillPath(p, &path);
}
VOID OnPaint(HDC hdc,int width, int height)
{
    Graphics graphics(hdc);

    graphics.SetSmoothingMode(SmoothingMode::SmoothingModeHighQuality);

    graphics.SetCompositingQuality(CompositingQuality::CompositingQualityInvalid);
    graphics.SetPixelOffsetMode(PixelOffsetMode::PixelOffsetModeHighQuality);
    SolidBrush mySolidBrush(Color(255, 255, 0, 0)); ;
    Gdiplus::Rect rect1;
    rect1.X = 0;
    rect1.Y = TOPEXTENDWIDTH;
    rect1.Width = width;
    rect1.Height = height- TOPEXTENDWIDTH-111;
    UINT8 rad[4]{ 0,12,12,0 };
    FillRoundRectangle(&graphics, &mySolidBrush, rect1, rad);
    SolidBrush DarkSolidBrush(Color(255, 0, 1, 0)); ;

    Gdiplus::Rect rectX = {0,455,55,55};
    graphics.FillEllipse(&DarkSolidBrush,rectX);
}


LRESULT CustomCaptionProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam, bool* pfCallDWP)
{
    LRESULT lRet = 0;
    HRESULT hr = S_OK;
    bool fCallDWP = true; // Pass on to DefWindowProc?
    static HICON hIcon = NULL;

    fCallDWP = !DwmDefWindowProc(hWnd, message, wParam, lParam, &lRet);
    if (message == WM_CREATE)
    {

        RECT rcClient;
        GetWindowRect(hWnd, &rcClient);
        // Inform the application of the frame change.
        SetWindowPos(hWnd, NULL, rcClient.left, rcClient.top, RECTWIDTH(rcClient), RECTHEIGHT(rcClient), SWP_FRAMECHANGED);

        HMODULE hDLL = LoadLibrary(L"Setupapi.dll");
        if (hDLL)
        {
            hIcon = (HICON)LoadImage(hDLL, MAKEINTRESOURCE(2), IMAGE_ICON, 32, 32, LR_DEFAULTCOLOR | LR_SHARED);
        }
        SetWindowLong(hWnd, GWL_EXSTYLE,
            GetWindowLong(hWnd, GWL_EXSTYLE) | WS_EX_LAYERED);
        /*Use pointer to function*/
        SetLayeredWindowAttributes(hWnd, 0,
            (255 * 70) / 100, LWA_ALPHA);
        fCallDWP = true;
        lRet = 0;
    }

    // Handle window activation.
    if (message == WM_ACTIVATE)
    {
        // Extend the frame into the client area.
        MARGINS margins;
        margins.cxLeftWidth = 0;
        margins.cxRightWidth = 0;
        margins.cyBottomHeight = 0;
        margins.cyTopHeight = 0;
        hr = DwmExtendFrameIntoClientArea(hWnd, &margins);

        if (!SUCCEEDED(hr))
        {
            // Handle error.
        }

        fCallDWP = true;
        lRet = 0;
    }

    if (message == WM_PAINT)
    {
        PAINTSTRUCT ps;
        BITMAP bm;
        RECT rect, rectCaptionButtonBounds, rectText,myRect,ContentRect,ClientRect,CaptionBorderBottom,  rect_EXIT_BTN, rect_RESTORE_BTN, rect_MINIMIZE_BTN;

        HFONT windowTitleText;

        GetClientRect(hWnd,&ClientRect);
        BeginPaint(hWnd, &ps);

        SetGraphicsMode(ps.hdc, GM_ADVANCED);

        SetLayeredWindowAttributes(hWnd, RGB(0, 0, 0), 255, LWA_COLORKEY);
        SetBkMode(ps.hdc, TRANSPARENT);
        if (SUCCEEDED(DwmGetWindowAttribute(hWnd, DWMWA_CAPTION_BUTTON_BOUNDS, &rectCaptionButtonBounds, sizeof(rectCaptionButtonBounds))))
        {
            GetClientRect(hWnd, &rect);
            
            //HRGN hrgn_cptBtmBrdrRND = CreateRoundRectRgn(0, 0, RECTWIDTH(ClientRect), RECTHEIGHT(ClientRect), 16, 16);

            //FillRgn(ps.hdc, hrgn_cptBtmBrdrRND, DARKBLUE_BRUSH);

            HRGN hrgn = CreateRectRgn(0, 0, RECTWIDTH(ClientRect), TOPEXTENDWIDTH);

            FillRgn(ps.hdc, hrgn, PURPLE_BRUSH);

            DrawIconEx(ps.hdc, rect.right - (rectCaptionButtonBounds.right - rectCaptionButtonBounds.left) - 32, 0, hIcon, 32, 32, 0, NULL, DI_NORMAL);
            SetRect(&myRect, LEFTEXTENDWIDTH, 10, RECTWIDTH(rect)-200, TOPEXTENDWIDTH);
            SetTextColor(ps.hdc, RGB(1, 0, 0));
            DrawText(ps.hdc,L"test",-1,&myRect, DT_SINGLELINE | DT_RIGHT);


            SetTextColor(ps.hdc, RGB(255, 255, 255));
            WCHAR wsText[255] = L"ARMNET";
            SetRect(&rectText, LEFTEXTENDWIDTH, 0, RECTWIDTH(rect), TOPEXTENDWIDTH);


                windowTitleText =
                CreateFontA
                (
                  32,
                  0,
                  GM_ADVANCED,
                  0,
                  FW_DONTCARE,
                  false,
                  false,
                  false,
                  DEFAULT_CHARSET,
                  OUT_OUTLINE_PRECIS,
                  CLIP_DEFAULT_PRECIS,
                  CLEARTYPE_QUALITY, //BETTER BLENDING THAN ANTIALIASED
                  VARIABLE_PITCH,
                  "RETRO COMPUTER");

                SelectObject(ps.hdc, windowTitleText);
            DrawText(ps.hdc, wsText, -1, &rectText, DT_SINGLELINE | DT_VCENTER);
            DeleteObject(windowTitleText);
            DeleteObject(hrgn);

        }

        //CONTENT AREA
        //SetRect(&ContentRect, 0, TOPEXTENDWIDTH, RECTWIDTH(ClientRect) - 0, RECTHEIGHT(ClientRect) - 0);
        //FillRect(ps.hdc, &ContentRect, DARKBLUE_BRUSH);

        HRGN hrgn_cptBtmBrdr = CreateRectRgn(0, TOPEXTENDWIDTH-1, RECTWIDTH(ClientRect), TOPEXTENDWIDTH);
        FillRgn(ps.hdc, hrgn_cptBtmBrdr, CreateSolidBrush(RGB(132, 68, 133)));

        hrgn_cptBtmBrdr = CreateRectRgn(0, TOPEXTENDWIDTH - 2, RECTWIDTH(ClientRect), TOPEXTENDWIDTH-1);
        FillRgn(ps.hdc, hrgn_cptBtmBrdr,CreateSolidBrush(RGB(185, 91, 186)));

        //BUTTONS
        hrgn_cptBtmBrdr = CreateRectRgn(RECTWIDTH(ClientRect)-32, 0, RECTWIDTH(ClientRect), 32);
        FillRgn(ps.hdc, hrgn_cptBtmBrdr, CreateSolidBrush(RGB(11, 11, 111)));
        hrgn_cptBtmBrdr = CreateRectRgn(RECTWIDTH(ClientRect) - 64, 0, RECTWIDTH(ClientRect)-32, 32);
        FillRgn(ps.hdc, hrgn_cptBtmBrdr, CreateSolidBrush(RGB(111, 11, 111)));
        hrgn_cptBtmBrdr = CreateRectRgn(RECTWIDTH(ClientRect) - 96, 0, RECTWIDTH(ClientRect)-64, 32);
        FillRgn(ps.hdc, hrgn_cptBtmBrdr, CreateSolidBrush(RGB(11, 111, 11)));
        
        OnPaint(ps.hdc, RECTWIDTH(ClientRect), RECTHEIGHT(ClientRect));

        DeleteObject(hrgn_cptBtmBrdr);
        EndPaint(hWnd, &ps);



        fCallDWP = true;
        lRet = 0;
    }

    // Handle the non-client size message.
    if ((message == WM_NCCALCSIZE) && (wParam == TRUE))
    {
        // Calculate new NCCALCSIZE_PARAMS based on custom NCA inset.
        NCCALCSIZE_PARAMS *pncsp = reinterpret_cast<NCCALCSIZE_PARAMS*>(lParam);

        pncsp->rgrc[0].left = pncsp->rgrc[0].left + 1;
        pncsp->rgrc[0].top = pncsp->rgrc[0].top + 0;
        pncsp->rgrc[0].right = pncsp->rgrc[0].right - 1;
        pncsp->rgrc[0].bottom = pncsp->rgrc[0].bottom - 1;

        lRet = 0;
        // No need to pass the message on to the DefWindowProc.
        fCallDWP = false;
    }

    // Handle hit testing in the NCA if not handled by DwmDefWindowProc.
    if ((message == WM_NCHITTEST) && (lRet == 0))
    {
        lRet = HitTestNCA(hWnd, wParam, lParam);

        if (lRet != HTNOWHERE)
        {
            fCallDWP = false;
        }
    }

    if (message == WM_SIZE)
    {
        if (unsigned int(wParam) == SIZE_MAXIMIZED) {

        }
        else
        {

        }
    }

    if (message == WM_GETMINMAXINFO)
    {
        LPMINMAXINFO lpMMI = (LPMINMAXINFO)lParam;
        lpMMI->ptMinTrackSize.x = 800;
        lpMMI->ptMinTrackSize.y = 600;
    }

    if (message == WM_DESTROY)
    PostQuitMessage(0);
    *pfCallDWP = fCallDWP;

    return lRet;
}

// Hit test the frame for resizing and moving.
LRESULT HitTestNCA(HWND hWnd, WPARAM wParam, LPARAM lParam)
{
    // Get the point coordinates for the hit test.
    POINT ptMouse = { GET_X_LPARAM(lParam), GET_Y_LPARAM(lParam) };

    // Get the window rectangle.
    RECT rcWindow;
    GetWindowRect(hWnd, &rcWindow);

    // Get the frame rectangle, adjusted for the style without a caption.
    RECT rcFrame = { 0 };
    AdjustWindowRectEx(&rcFrame, WS_OVERLAPPEDWINDOW & ~WS_CAPTION, FALSE, NULL);

    // Determine if the hit test is for resizing. Default middle (1,1).
    USHORT uRow = 1;
    USHORT uCol = 1;
    bool fOnResizeBorder = false;

    // Determine if the point is at the top or bottom of the window.
    if ((ptMouse.y >= rcWindow.top && ptMouse.y < rcWindow.top + TOPEXTENDWIDTH)  )
    {
        if((ptMouse.x < rcWindow.right - 100) || (ptMouse.y > rcWindow.top + 32)){
        fOnResizeBorder = (ptMouse.y < (rcWindow.top - rcFrame.top));
        uRow = 0;
        }
    }

    else if (ptMouse.y < rcWindow.bottom && ptMouse.y >= rcWindow.bottom - BOTTOMEXTENDWIDTH)
    {
        uRow = 2;
    }

    // Determine if the point is at the left or right of the window.
    if (ptMouse.x >= rcWindow.left && ptMouse.x < rcWindow.left + LEFTEXTENDWIDTH)
    {
        uCol = 0; // left side
    }
    else if (ptMouse.x < rcWindow.right && ptMouse.x >= rcWindow.right - RIGHTEXTENDWIDTH)
    {
        uCol = 2; // right side
    }

    // Hit test (HTTOPLEFT, ... HTBOTTOMRIGHT)
    LRESULT hitTests[3][3] =
    {
        { fOnResizeBorder ? HTTOPLEFT : HTLEFT, fOnResizeBorder ? HTTOP : HTCAPTION, fOnResizeBorder ? HTTOPRIGHT : HTRIGHT },
        { HTLEFT,       HTNOWHERE,     HTRIGHT },
        { HTBOTTOMLEFT, HTBOTTOM, HTBOTTOMRIGHT },
    };

    return hitTests[uRow][uCol];
}

LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    bool fCallDWP = true;
    BOOL fDwmEnabled = FALSE;
    LRESULT lRet = 0;
    HRESULT hr = S_OK;

    // Winproc worker for custom frame issues.
    hr = DwmIsCompositionEnabled(&fDwmEnabled);
    if (SUCCEEDED(hr))
    {
        lRet = CustomCaptionProc(hWnd, message, wParam, lParam, &fCallDWP);
    }

    // Winproc worker for the rest of the application.
    if (fCallDWP)
    {
        //  lRet = AppWinProc(hWnd, message, wParam, lParam);
        lRet = DefWindowProc(hWnd, message, wParam, lParam);
    }
    return lRet;
}

Solution

  • Well, after searching for solutions, in the end I think I found an answer.

    SOLUTION: A solution for Anti aliasing issue could be capturing the background of entire window bound by exluding it from capture. At this time you will not need layered window since you are painting the background with a bitmap. HDC will be able to blend it with background. Hopefully Windows 10 2004 version you have an option called:

    WDA_EXCLUDEFROMCAPTURE

    USAGE:

    SetWindowDisplayAffinity(hWnd, WDA_EXCLUDEFROMCAPTURE); //At creation time

    SOURCE: https://blogs.windows.com/windowsdeveloper/2019/09/16/new-ways-to-do-screen-capture/

    After that you can paint background with bitmap and then paint everything over it. However, this produced low performance and I did not benefit. Still, that works and produces ANTI-ALIASED looking when drawn. For the performance issue Direct2D can be used for drawing from returned bitmap.

    EXAMPLE:

    int DsktpBkSS(HWND hWnd) {
    
        HDC hdcScreen;
        HDC hdcWindow;
        HDC hdcMemDC = NULL;
        HBITMAP hbmScreen = NULL;
        BITMAP bmpScreen;
        RECT windowPos;
        SetWindowDisplayAffinity(hWnd, WDA_EXCLUDEFROMCAPTURE);
        GetWindowRect(hWnd, &windowPos);
    
        // Retrieve the handle to a display device context for the client 
        // area of the window. 
        hdcScreen = GetDC(NULL);
        hdcWindow = GetDC(hWnd);
    
        // Create a compatible DC which is used in a BitBlt from the window DC
        hdcMemDC = CreateCompatibleDC(hdcWindow);
    
        if (!hdcMemDC)
        {
            MessageBox(hWnd, L"CreateCompatibleDC has failed", L"Failed", MB_OK);
        }
    
        // Get the client area for size calculation
        RECT rcClient;
        GetClientRect(hWnd, &rcClient);
    
        //This is the best stretch mode
        SetStretchBltMode(hdcWindow, HALFTONE);
    
        //The source DC is the entire screen and the destination DC is the current window (HWND)
        if (!StretchBlt(hdcWindow,
            0, 0,
            GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN),
            hdcScreen,
            windowPos.left+1,windowPos.top,
            GetSystemMetrics(SM_CXSCREEN),
            GetSystemMetrics(SM_CYSCREEN),
            SRCCOPY))
        {
            MessageBox(hWnd, L"StretchBlt has failed", L"Failed", MB_OK);
        }
    
        // Create a compatible bitmap from the Window DC
        hbmScreen = CreateCompatibleBitmap(hdcWindow, rcClient.right - rcClient.left, rcClient.bottom - rcClient.top);
    
        if (!hbmScreen)
        {
            MessageBox(hWnd, L"CreateCompatibleBitmap Failed", L"Failed", MB_OK);
        }
    
        // Select the compatible bitmap into the compatible memory DC.
        if(hdcMemDC && hbmScreen){
        SelectObject(hdcMemDC, hbmScreen);
        }
        // Bit block transfer into our compatible memory DC.
        if(hdcMemDC)
        if (!BitBlt(hdcMemDC,
            0, 0,
            rcClient.right - rcClient.left, rcClient.bottom - rcClient.top,
            hdcWindow,
            0, 0,
            SRCCOPY))
        {
            MessageBox(hWnd, L"BitBlt has failed", L"Failed", MB_OK);
        }
    
        // Get the BITMAP from the HBITMAP
        if(hbmScreen)
        GetObjectW(hbmScreen, sizeof(BITMAP), &bmpScreen);
    
        if (hbmScreen)DeleteObject(hbmScreen);
        if (hdcMemDC)DeleteObject(hdcMemDC);
        ReleaseDC(NULL, hdcScreen);
        ReleaseDC(hWnd, hdcWindow);
    
        return 0;
    }
    

    Later,

    (...){
    if(message==WM_PAINT)
    {
    PAINTSTRUCT ps;
    BeginPaint(hWnd, &ps);
    Graphics graphics(ps.hdc);
    /*SET SMOOTHING (AA)*/
    graphics.SetSmoothingMode(SmoothingMode::SmoothingModeHighQuality);
    if(DsktpBkSS(hWnd))
    {
    /*DRAW ROUNDED RECTANGLE*/
    }
    }
    //...
    };
    

    One more important thing for window drawn with "Desktop Window Manager" when resizing from left too fast and continuously that will produce "flickering" after a time. On Microsoft docs that is recommended that to use StretchBlt(...) for drawing since GDI+ cause this.

    flicker mentioned for DWM: https://learn.microsoft.com/en-us/windows/win32/api/wingdi/nf-wingdi-stretchblt

    "Use BitBlt or StretchBlt function instead of Windows GDI+ to present your drawing for rendering. GDI+ renders one scan line at a time with software rendering. This can cause flickering in your applications."