I'm writing a program to rapidly capture images from one window, modify them, and output them to another window using Xlib with C++. I have this working via XGetImage:
#include <iostream>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
const int TEST_SIZE = 512;
int main()
{
// Open default display
Display *display = XOpenDisplay(nullptr);
int screen = DefaultScreen(display);
Window rootWin = RootWindow(display, screen);
GC graphicsContext = DefaultGC(display, screen);
// Create new window and subscribe to events
long blackPixel = BlackPixel(display, screen);
long whitePixel = WhitePixel(display, screen);
Window newWin = XCreateSimpleWindow(display, rootWin, 0, 0, TEST_SIZE, TEST_SIZE, 1, blackPixel, whitePixel);
XMapWindow(display, newWin);
XSelectInput(display, newWin, ExposureMask | KeyPressMask);
// Main event loop for new window
XImage *image;
XEvent event;
bool exposed = false;
bool killWindow = false;
while (!killWindow)
{
// Handle pending events
if (XPending(display) > 0)
{
XNextEvent(display, &event);
if (event.type == Expose)
{
exposed = true;
} else if (event.type == NoExpose)
{
exposed = false;
} else if (event.type == KeyPress)
{
killWindow = true;
}
}
// Capture the original image
image = XGetImage(display, rootWin, 0, 0, TEST_SIZE, TEST_SIZE, AllPlanes, ZPixmap);
// Modify the image
if (image->data != nullptr)
{
long pixel = 0;
for (int x = 0; x < image->width; x++)
{
for (int y = 0; y < image->height; y++)
{
// Invert the color of each pixel
pixel = XGetPixel(image, x, y);
XPutPixel(image, x, y, ~pixel);
}
}
}
// Output the modified image
if (exposed && killWindow == false)
{
XPutImage(display, newWin, graphicsContext, image, 0, 0, 0, 0, TEST_SIZE, TEST_SIZE);
}
XDestroyImage(image);
}
// Goodbye
XCloseDisplay(display);
}
It generates output like this: https://streamable.com/hovg9
I'm happy to have gotten this far, but from what I've read the performance isn't going to scale very well because it has to allocate space for a new image at every frame. In fact, without the call to XDestroyImage at the end of the loop this program fills up all 16GB of memory on my machine in a matter of seconds!
It seems the recommended approach here is to to set up a shared memory space where X can write the contents of each frame and my program can subsequently read and modify them without the need for any extra allocation. Since the call to XShmGetImage blocks and waits for IPC I believe this means I won't have to worry about any concurrency issues in the shared space.
I've attempted to implement the XShmGetImage approach with the following code:
#include <iostream>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <X11/Xlib.h>
#include <X11/extensions/XShm.h>
#include <X11/Xutil.h>
const int TEST_SIZE = 512;
int main()
{
// Open default display
Display *display = XOpenDisplay(nullptr);
int screen = DefaultScreen(display);
Window rootWin = RootWindow(display, screen);
GC graphicsContext = DefaultGC(display, screen);
// Create new window and subscribe to events
long blackPixel = BlackPixel(display, screen);
long whitePixel = WhitePixel(display, screen);
Window newWin = XCreateSimpleWindow(display, rootWin, 0, 0, TEST_SIZE, TEST_SIZE, 1, blackPixel, whitePixel);
XMapWindow(display, newWin);
XSelectInput(display, newWin, ExposureMask | KeyPressMask);
// Allocate shared memory for image capturing
Visual *visual = DefaultVisual(display, 0);
XShmSegmentInfo shminfo;
int depth = DefaultDepth(display, screen);
XImage *image = XShmCreateImage(display, visual, depth, ZPixmap, nullptr, &shminfo, TEST_SIZE, TEST_SIZE);
shminfo.shmid = shmget(IPC_PRIVATE, image->bytes_per_line * image->height, IPC_CREAT | 0666);
shmat(shminfo.shmid, nullptr, 0);
shminfo.shmaddr = image->data;
shminfo.readOnly = False;
XShmAttach(display, &shminfo);
// Main event loop for new window
XEvent event;
bool exposed = false;
bool killWindow = false;
while (!killWindow)
{
// Handle pending events
if (XPending(display) > 0)
{
XNextEvent(display, &event);
if (event.type == Expose)
{
exposed = true;
} else if (event.type == NoExpose)
{
exposed = false;
} else if (event.type == KeyPress)
{
killWindow = true;
}
}
// Capture the original image
XShmGetImage(display, rootWin, image, 0, 0, AllPlanes);
// Modify the image
if (image->data != nullptr) // NEVER TRUE. DATA IS ALWAYS NULL!
{
long pixel = 0;
for (int x = 0; x < image->width; x++)
{
for (int y = 0; y < image->height; y++)
{
// Invert the color of each pixel
pixel = XGetPixel(image, x, y);
XPutPixel(image, x, y, ~pixel);
}
}
}
// Output the modified image
if (exposed && killWindow == false)
{
XShmPutImage(display, newWin, graphicsContext, image, 0, 0, 0, 0, 512, 512, false);
}
}
// Goodbye
XFree(image);
XCloseDisplay(display);
}
Somehow, the images are still being captured and sent to the new window, but the data pointer within the XImage is always null. I can't modify any of the contents and any usage of the XGetPixel and XPutPixel macros will fail. Notice in this video that none of the colors are being inverted like before: https://streamable.com/dckyv
This doesn't make any sense to me. Clearly the data is still being transferred between the windows, but where is it in the XImage structure? How can I access it via code?
It turns out I wasn't reading the signature for shmat correctly. It doesn't have a void return type, it returns a void pointer. This needs to be assigned directly to the XImage's data pointer in order for the data pointer to do anything.
In the call to XShmPutImage the shared memory location is referenced internally rather than the XImage's data pointer which is why this was still working even without utilizing the return value from shmat.
Here's the working code, along with a few other additions which I made for error handling and teardown:
#include <iostream>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <X11/Xlib.h>
#include <X11/extensions/XShm.h>
#include <X11/Xutil.h>
const int TEST_SIZE = 512;
static int handleXError(Display *display, XErrorEvent *event)
{
printf("XErrorEvent triggered!\n");
printf("error_code: %d", event->error_code);
printf("minor_code: %d", event->minor_code);
printf("request_code: %d", event->request_code);
printf("resourceid: %lu", event->resourceid);
printf("serial: %d", event->error_code);
printf("type: %d", event->type);
return 0;
}
int main()
{
// Open default display
XSetErrorHandler(handleXError);
Display *display = XOpenDisplay(nullptr);
int screen = DefaultScreen(display);
Window rootWin = RootWindow(display, screen);
GC graphicsContext = DefaultGC(display, screen);
// Create new window and subscribe to events
long blackPixel = BlackPixel(display, screen);
long whitePixel = WhitePixel(display, screen);
Window newWin = XCreateSimpleWindow(display, rootWin, 0, 0, TEST_SIZE, TEST_SIZE, 1, blackPixel, whitePixel);
XMapWindow(display, newWin);
XSelectInput(display, newWin, ExposureMask | KeyPressMask);
// Allocate shared memory for image capturing
XShmSegmentInfo shminfo;
Visual *visual = DefaultVisual(display, screen);
int depth = DefaultDepth(display, screen);
XImage *image = XShmCreateImage(display, visual, depth, ZPixmap, nullptr, &shminfo, TEST_SIZE, TEST_SIZE);
shminfo.shmid = shmget(IPC_PRIVATE, image->bytes_per_line * image->height, IPC_CREAT | 0777);
image->data = (char*)shmat(shminfo.shmid, 0, 0);
shminfo.shmaddr = image->data;
shminfo.readOnly = False;
XShmAttach(display, &shminfo);
XSync(display, false);
shmctl(shminfo.shmid, IPC_RMID, 0);
// Main event loop for new window
XEvent event;
bool exposed = false;
bool killWindow = false;
while (!killWindow)
{
// Handle pending events
if (XPending(display) > 0)
{
XNextEvent(display, &event);
if (event.type == Expose)
{
exposed = true;
} else if (event.type == NoExpose)
{
exposed = false;
} else if (event.type == KeyPress)
{
killWindow = true;
}
}
// Capture the original image
XShmGetImage(display, rootWin, image, 0, 0, AllPlanes);
// Modify the image
if(image->data != nullptr) // NEVER TRUE. DATA IS ALWAYS NULL!
{
long pixel = 0;
for (int x = 0; x < image->width; x++)
{
for (int y = 0; y < image->height; y++)
{
// Invert the color of each pixel
pixel = XGetPixel(image, x, y);
XPutPixel(image, x, y, ~pixel);
}
}
}
// Output the modified image
if (exposed && killWindow == false)
{
XShmPutImage(display, newWin, graphicsContext, image, 0, 0, 0, 0, 512, 512, false);
}
}
// Goodbye
XShmDetach(display, &shminfo);
XDestroyImage(image);
shmdt(shminfo.shmaddr);
XCloseDisplay(display);
}