Boost.Process v2: How to asynchronously read output and also check for termination

I'm spawning a process and reading its output using Boost.Process v2 and C++ 20 coroutines like this:

boost::asio::io_context gContext;
namespace bp = boost::process::v2;
struct Subprocess {
  bp::process process;
  std::string output;
};

// ...

auto pipe_stdout = std::make_unique<boost::asio::readable_pipe>(gContext);
auto pipe_stderr = std::make_unique<boost::asio::readable_pipe>(gContext);
auto subprocess = std::make_unique<Subprocess>(Subprocess{
    bp::process(gContext, "/bin/sh", { "-c", "my command" },
                bp::process_stdio{ nullptr, *pipe_stdout, *pipe_stderr }),
    std::string{} });
for (auto* pipe : { &pipe_stdout, &pipe_stderr }) {
  boost::asio::co_spawn(
      gContext,
      [pipe = std::move(*pipe),
       output = &subprocess->output]() -> boost::asio::awaitable<void> {
        while (true) {
          std::array<char, 1024> buf;
          size_t len = co_await pipe->async_read_some(
              boost::asio::buffer(buf), boost::asio::use_awaitable);
          if (len == 0 && !pipe->is_open()) {
            co_return;
          }
          output->append(buf.data(), len);
        }
      },
      boost::asio::detached);
}

I also set up a handler which collects the process if it finished (finished_ and running_ are of type std::vector<std::unique_ptr<Subprocess>>:

subprocess->process.async_wait(
    [this, p = subprocess.get()](bp::error_code ec, int exit_code) {
      assert(!ec);
      auto it = std::ranges::find_if(
        running_, [p](const auto& up) { return up.get() == p; });
      finished_.emplace_back(std::move(*it));
      running_.erase(it);
    });

I then have a mainloop which checks if a process has finished and if so, processes the output:

while (true) {
  while (!running_.empty() && finished_.empty()) {
    gContext.run_one();
  }
  if (finished_.empty()) {
    continue;
  }
  // process finished_.back(), get its output, etc.
  finished_.pop_back();
}

The issue I have now is that sometimes I don't catch all the output, it seems there's still some left in a buffer and my coroutine reading the pipes hasn't finished. How can I avoid that race? Should I somehow read the output and await the exit in the same coroutine?

Solution

DISCLAIMER There are serious security concerns running commands like this. My demonstration assumes you can trust the source of the commands. Do not use my code as is!

Trying to piece this together, I figured it would be easier to show how I'd write it instead:

Live On Coliru

#include <boost/asio.hpp>
#include <boost/asio/experimental/awaitable_operators.hpp>
#include <boost/process/v1/group.hpp>
#include <boost/process/v2.hpp>
#include <iostream>
namespace asio = boost::asio;
namespace bp = boost::process::v2;

asio::awaitable<std::string> async_command_output(std::string command) {
    auto ex = co_await asio::this_coro::executor;

    asio::readable_pipe pout(ex), perr(ex);
    bp::process         child{
        ex, "/bin/sh", {"-c", command}, bp::process_stdio{.in = nullptr, .out = pout, .err = perr}};

    std::string output;

    auto read_loop = [&output](asio::readable_pipe& p) -> asio::awaitable<void> {
        for (std::array<char, 1024> buf;;) {
            auto [ec, n] = co_await p.async_read_some(asio::buffer(buf), asio::as_tuple(asio::deferred));
            if (n)
                output.append(buf.data(), n);

            if (ec) {
                std::cerr << "read_loop: " << ec.message() << std::endl;
                break; // or co_return;
            }
        }
    };

    using namespace asio::experimental::awaitable_operators;
    int exit_code = co_await (                //
        read_loop(pout) &&                    //
        read_loop(perr) &&                    //
        child.async_wait(asio::use_awaitable) //
    );

    std::cerr << "Command returned exit code " << exit_code << std::endl;

    co_return output;
}

int main(int argc, char** argv) {
    asio::io_context ioc;

    co_spawn(
        ioc,
        [cmd = argc > 1 ? argv[1] : "ls -l"] -> asio::awaitable<void> {
            std::string output = co_await async_command_output(cmd);
            std::cout << "Output: " << quoted(output) << std::endl;
        },
        asio::detached);

    ioc.run();
}

Demo runs (click for high res):