Linux, how to capture screen, and simulate mouse movements

Question

I need to capture screen (as print screen) in the way so I can access pixel color data, to do some image recognition, after that I will need to generate mouse events on the screen such as left click, drag and drop (moving mouse while button is pressed, and then release it). Once its done, image will be deleted.

Note: I need to capture whole screen everything that user can see, and I need to simulate clicks outside window of my program (if it makes any difference)

Spec: Linux ubuntu Language: C++

Performance is not very important,"print screen" function will be executed once every ~10 sec. Duration of the process can be up to 24 hours so method needs to be stable and memory leaks free (as usuall :)

I was able to do in windows with win GDI and some windows events, but I'ev no idea how to do it in Linux.

Thanks a lot

Answer 1

//sg

//Solution using Xlib for those who use Linux
#include <X11/Xlib.h>
#include<stdio.h>
#include<unistd.h>
#include <stdlib.h>
#include <string.h>

#include <unistd.h>

#include <X11/Xlib.h>
#include <X11/Xutil.h>

void mouseClick(int button)
{
    Display *display = XOpenDisplay(NULL);

    XEvent event;

    if(display == NULL)
    {
        fprintf(stderr, "Cannot initialize the display\n");
        exit(EXIT_FAILURE);
    }

    memset(&event, 0x00, sizeof(event));

    event.type = ButtonPress;
    event.xbutton.button = button;
    event.xbutton.same_screen = True;

    XQueryPointer(display, RootWindow(display, DefaultScreen(display)), &event.xbutton.root, &event.xbutton.window, &event.xbutton.x_root, &event.xbutton.y_root, &event.xbutton.x, &event.xbutton.y, &event.xbutton.state);

    event.xbutton.subwindow = event.xbutton.window;

    while(event.xbutton.subwindow)
    {
        event.xbutton.window = event.xbutton.subwindow;

        XQueryPointer(display, event.xbutton.window, &event.xbutton.root, &event.xbutton.subwindow, &event.xbutton.x_root, &event.xbutton.y_root, &event.xbutton.x, &event.xbutton.y, &event.xbutton.state);
    }

    if(XSendEvent(display, PointerWindow, True, 0xfff, &event) == 0) fprintf(stderr, "Error\n");

    XFlush(display);

    usleep(100000);

    event.type = ButtonRelease;
    event.xbutton.state = 0x100;

    if(XSendEvent(display, PointerWindow, True, 0xfff, &event) == 0) fprintf(stderr, "Error\n");

    XFlush(display);

    XCloseDisplay(display);
}
int main(int argc,char * argv[]) {

    int x , y;
    x=atoi(argv[1]);
    y=atoi(argv[2]);
    Display *display = XOpenDisplay(0);

    Window root = DefaultRootWindow(display);
    XWarpPointer(display, None, root, 0, 0, 0, 0, x, y);
    mouseClick(Button1);
    XFlush(display);
    XCloseDisplay(display);
    return 0;
}

Build it and then to simulate a click at x ,y do:

$ ./a.out x y

i.e.

$ g++ -lX11 sgmousesim2.cpp

$ ./a.out 123 13

Just in case you are still interested.

Answer 2

Swinput is a solution for simulating mouse/key events. You need to compile it probably for your kernel. Xorg provided some headers for recording mouse/key events but I think it is broken at the moment. There is a C code evtest that can be used to capture events from /dev/input/eventX, /dev/input/mice files. It can be helpful.

Edit:

The bug was fixed in Xorg record extension, so it may be working too.

Answer 3

For it to work correctly, you must call XFlush after you warp the pointer

Tested on linux Mint19 (Cinamon 3.8) XWarpPointer(display, None, root, 0, 0, 0, 0, x, y); XFlush(display);

Answer 4

I couldn't get the clicking working with the method @axiom used, only movement of the pointer. I used this instead: (Ubuntu 18.04).

Compiles with: g++ mouse_click.cpp -lX11 -lXtst -lstdc++

#include<stdio.h>
#include<unistd.h>
#include <stdlib.h>
#include <string.h>

#include <unistd.h>

#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <X11/extensions/XTest.h>


void mouseClick(int button)
{
    Display *display = XOpenDisplay(NULL);

    // click left button
    XTestFakeButtonEvent(display, Button1, true, 0);
    XFlush(display);

    usleep(10000);

    // release left mouse
    XTestFakeButtonEvent(display, Button1, false, 0);
    XFlush(display);


    XCloseDisplay(display);


}
int main(int argc,char * argv[]) {

    int x , y;
    x=atoi(argv[1]);
    y=atoi(argv[2]);
    Display *display = XOpenDisplay(0);

    Window root = DefaultRootWindow(display);
    XTestFakeMotionEvent(display, root, x, y, 0);
    XFlush(display);
    mouseClick(Button1);
    XFlush(display);
    XCloseDisplay(display);
    return 0;
}