Xlib. Check if window is overlapped

I need to check (in my C++ application) if application's window is overlapped (partially or completely) by another window. How can I get this info using xlib?

I tried get WM_STATE property of the window and compare it with NormalState. However, when I overlap target window my condition (state != NormalState) wasn't executed. I also tried to compare _NET_WM_STATE property with _NET_WM_STATE_HIDDEN and this method didn't give needed result.

I tried to implement suggested by spectras solution, and now it looks like working variant(Note: GetWindowProperty is my wrapper around corresponding Xlib function):

bool ApplicationHelper::IsWindowOverlapped(const void* hWnd, _XDisplay* pd)
{
    Display* pDisplay = pd == nullptr ? XOpenDisplay(nullptr) : pd;
    if(pDisplay == nullptr)
    {
        return true;
    }
    auto root = XDefaultRootWindow(pDisplay);
    Window parent;
    /*Window* windowList;
    unsigned nchildren;
    if (!XQueryTree(pDisplay, root, &root, &parent, &windowList, &nchildren))
    {
        if(pd == nullptr)
        {
            XCloseDisplay(pDisplay);
        }
        return true;
    }*/

    Atom propCleints = XInternAtom(pDisplay, "_NET_CLIENT_LIST_STACKING", True);
    unsigned long ulBytesReturned = 0;
    Window *windowList = (Window *)GetWindowProperty(pDisplay, root, propCleints, &ulBytesReturned);
    unsigned long nchildren = ulBytesReturned / sizeof(Window);
    int32_t actualDesktop = GetWindowDesktop(pDisplay, (TWindow) hWnd);

    WindowRect targetWindowRect;
    GetWindowRect(hWnd, targetWindowRect, pDisplay);
    GdkRectangle targetGdkRect;
    targetGdkRect.x = targetWindowRect.x;
    targetGdkRect.y = targetWindowRect.y;
    targetGdkRect.width = targetWindowRect.width;
    targetGdkRect.height = targetWindowRect.height;
    bool handleWindow = false;
    bool res = false;
    for (unsigned long i = 0; i < nchildren; ++i)
    {
        auto window = windowList[i];
        if((Window)hWnd == window)
        {
            handleWindow = true;
            continue;
        }
        if(handleWindow)
        {
            if((GetWindowDesktop(pDisplay, window) != actualDesktop) || IsWindowHidden((void*)window, pDisplay))
            {
                continue;
            }
            else
            {
                WindowRect rc;
                GetWindowRect((void*)window, rc, pDisplay);
                GdkRectangle gdkRect;
                gdkRect.x = rc.x;
                gdkRect.y = rc.y;
                gdkRect.width = rc.width;
                gdkRect.height = rc.height;
                if(gdk_rectangle_intersect(&targetGdkRect, &gdkRect, nullptr))
                {
                    res = true;
                    break;
                }
            }
        }
    }
    XFree(windowList);
    if(pd == nullptr)
    {
        XCloseDisplay(pDisplay);
    }

    return res;
}

This code always returns true.

Solution

It's normal, as its name says, that property only signals whether the window is being hidden (for instance because it is minimized).

If you want to check for other windows above yours and hiding part of it, you'll have to do it manually.

For instance, this gets a list of all top-level windows:

auto root = XDefaultRootWindow(display);
Window parent;
Window * children;
unsigned nchildren;
if (!XQueryTree(display, root, &root, &parent, &children, &nchildren)) {
    std::cout <<"Failed to query top level windows list\n";
    return 1;
}

for (unsigned idx = 0; idx < nchildren; ++idx) {
    auto window = children[idx];
    // do something with window
}

XFree(children);

You will need to:

Find out where your own window is in the list. Any window that comes after it is above in the stacking order.
Query the desktop of each window above yours (_NET_WM_DESKTOP atom), eliminate any window not show on current desktop.
Query the display state of each window above yours (_NET_WM_STATE atom, looking for _NET_WM_STATE_HIDDEN atom in the list).
Query the size of each shown window above yours. Possibly using the Shape extension if you want to account for non-rectangular windows
Check whether any of those intersect with your own display area.