Is it possible to click inside a self made web app?

Question

I created a PyQt5 web app that navigates to https://www.google.com/ with the following code :

from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtWebEngineWidgets import *

import sys

class MainWindow(QMainWindow):

    def __init__(self, *args, **kwargs):
        super(MainWindow, self).__init__(*args, **kwargs)

        self.setWindowTitle("My Window")

        self.browser = QWebEngineView()
        self.browser.setUrl( QUrl("www.google.com") )
        self.setCentralWidget(self.browse) 

app = QApplication(sys.argv)

window = MainWindow()
window.show()

app.exec()

The idea is that I want to manipulate clicks inside the application, therefore on the google site, for example: moving the mouse to certain locations(x, y), without influencing the PC itself, like pyautogui does, meaning that I can run a code that randomly clicks inside the web app while I am able to do normal things on the PC, without affecting the mouse.

I will be glad to know if this is possible or not. And if it is, how?

Note : I don't really have experience in python

Answer 1

You can emulate the click by sending QMouseEvent, in the case of QWebEngineView the click should be issued to an internal widget that is part of the private Qt API but with a small logic can be accessed: using findChildren and the QMetaObject.

In the following example, click on position 400, 200 which in my case is a google banner that will open respective information.

import sys
from PyQt5 import QtCore, QtGui, QtWidgets, QtWebEngineWidgets


class MainWindow(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)

        self.setWindowTitle("My Window")

        self.browser = QtWebEngineWidgets.QWebEngineView()
        self.browser.setUrl(QtCore.QUrl("https://www.google.com/"))
        self.setCentralWidget(self.browser)
        self.browser.loadFinished.connect(self.on_loadFinished)

    @QtCore.pyqtSlot(bool)
    def on_loadFinished(self, ok):
        if not ok:
            return
        w = None
        for child in self.browser.findChildren(QtWidgets.QWidget):
            if (
                child.metaObject().className()
                == "QtWebEngineCore::RenderWidgetHostViewQtDelegateWidget"
            ):
                w = child
                break
        if w is not None:
            self.emulate_click(w, QtCore.QPoint(400, 200))

    def emulate_click(self, widget, pos):
        event_press = QtGui.QMouseEvent(
            QtCore.QEvent.MouseButtonPress,
            pos,
            QtCore.Qt.LeftButton,
            QtCore.Qt.LeftButton,
            QtCore.Qt.NoModifier,
        )
        QtCore.QCoreApplication.postEvent(widget, event_press)
        event_release = QtGui.QMouseEvent(
            QtCore.QEvent.MouseButtonRelease,
            pos,
            QtCore.Qt.LeftButton,
            QtCore.Qt.LeftButton,
            QtCore.Qt.NoModifier,
        )
        QtCore.QCoreApplication.postEvent(widget, event_release)


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    w = MainWindow()
    w.resize(640, 480)
    w.show()
    sys.exit(app.exec_())

If you have more information such as id or xpath you can click using javascript:

import sys
from PyQt5 import QtCore, QtGui, QtWidgets, QtWebEngineWidgets


class MainWindow(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)

        self.setWindowTitle("My Window")

        self.browser = QtWebEngineWidgets.QWebEngineView()
        self.browser.setUrl(QtCore.QUrl("https://www.google.com/"))
        self.setCentralWidget(self.browser)
        self.browser.loadFinished.connect(self.on_loadFinished)

    @QtCore.pyqtSlot(bool)
    def on_loadFinished(self, ok):
        if not ok:
            return
        xpath = r"//body[@id='gsr']/div[@id='viewport']/div[@id='main']/span[@id='body']/center/div[@id='lga']/div[@id='hplogo']/a/img[1]"
        s = (
            """
            (function() {
                var banner = document.evaluate("%s", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
                banner.click()
            })()"""
            % xpath
        )
        self.browser.page().runJavaScript(s)


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    w = MainWindow()
    w.resize(640, 480)
    w.show()
    sys.exit(app.exec_())