Run a number of groups of QProcess' after each other

Question

I need to call an external program N number of times. I'd like to do this in parallel. So my strategy thus far has been to start N QProcesses and keep a count of ones that have started and ones that have finished. (So I can figure out when they have all finished).

However, the external program takes up a fair bit of RAM, so I do not want more the 4 parallel processes at any one time.

What is a good strategy for this?

I don't think signals/slots is enough to achieve this (I can't think of a method that isn't highly convoluted)...perhaps I can do something with a queue?

How can I ensure I only have 4 processes running at any one time? How can I then figure out when all N processes have finally finished?

(Answers for pyside/pyqt preferred, but C++ is Ok)

Answer 1

Proof-of-concept:

h file

#ifndef CPROCESSRUNNER_H
#define CPROCESSRUNNER_H

#include <QObject>
#include <QQueue>

class QProcess;

class CProcessRunner : public QObject
{
    Q_OBJECT
    Q_PROPERTY(int processCount READ processCount WRITE setProcessCount)
public:
    explicit CProcessRunner(QObject *parent = 0);
    ~CProcessRunner();

    void addProcess(const QString& program);

    int processCount() const;

public slots:
    void setProcessCount(int arg);

private slots:
    void startFreeProcesses();

private:
    int getActiveProcessCount() const;

    QQueue<QProcess*> m_processes;
    int m_processCount;
};

#endif // CPROCESSRUNNER_H

cpp file

#include "CProcessRunner.h"
#include <QProcess>
CProcessRunner::CProcessRunner(QObject *parent)
    : QObject(parent)
    , m_processCount(0)
{

}

CProcessRunner::~CProcessRunner()
{

}

void CProcessRunner::addProcess(const QString& program)
{
    QProcess* pProcess = new QProcess(this);
    pProcess->setObjectName(program);
    m_processes.enqueue(pProcess);
    startFreeProcesses();
}

int CProcessRunner::processCount() const
{
    return m_processCount;
}

void CProcessRunner::setProcessCount(int arg)
{
    m_processCount = arg;
}

void CProcessRunner::startFreeProcesses()
{
    while (!m_processes.isEmpty() && (getActiveProcessCount() < m_processCount)) {
        QProcess* pProcess = m_processes.dequeue();
        connect(pProcess, SIGNAL(finished(int)), this, SLOT(startFreeProcesses()));
        connect(pProcess, SIGNAL(finished(int)), pProcess, SLOT(deleteLater()));

        pProcess->start(pProcess->objectName());
        pProcess->waitForStarted(-1);
    }
}

int CProcessRunner::getActiveProcessCount() const
{
    int result = 0;
    foreach (QProcess* pProcess, findChildren<QProcess*>()) {
        if (pProcess->state() == QProcess::Running) {
            result++;
        }
    }
    return result;
}