Asynchronously download multiple files using curl library with C++

Question

I want to write a program that will download a number of files asynchronously. I managed to download one file using the curl_easy interface, but for some reason the curl_multi interface does not download files at all. What am I doing wrong?

There's my functions (download_file is working as expected, download_files just creates empty files, but doesn't download them):

sample input:

file_names = {"text1.txt","text2.txt","text3.txt"}; file_path = R"(C:\Downloads\\)";
url = "http://downloadPage.com/"

static size_t write_data(const char *ptr, size_t size, size_t nmemb, std::ofstream *userdata);

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url);

bool download_file(const std::string &file_name, const std::string &file_path, const char *url) {
    CURL *curl;
    CURLcode res;
    curl = curl_easy_init();
    if (curl) {
        static std::ofstream file_write((file_path + file_name), std::ios::binary);

        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &file_write);
        curl_easy_perform(curl);
        curl_easy_cleanup(curl);

        file_write.close();

    } else {
        std::cout << "curl error!" << std::endl;
    }
    return res;
}

void download_files(std::vector<std::string> file_names, const std::string &file_path, const char *url,
                    int HANDLECOUNT) {
    CURLM *multi_handle;
    CURLMsg *msg;
    unsigned int transfers = 0;
    int left = 0;
    int msgs_left = -1;
    int i = 0;

    curl_global_init(CURL_GLOBAL_ALL);
    multi_handle = curl_multi_init();

    curl_multi_setopt(multi_handle, CURLMOPT_MAXCONNECTS, 8);

    int still_running = 1; /* keep number of running handles */

    /* Allocate one CURL handle per transfer */
    for (transfers = 0; transfers < 8 && transfers < HANDLECOUNT; transfers++)
        add_transfer(multi_handle, transfers, &left, file_names, file_path, url);

    do {
        int still_alive = 1;
        curl_multi_perform(multi_handle, &still_alive);

        while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
            if (msg->msg == CURLMSG_DONE) {
                char *url2;
                CURL *e = msg->easy_handle;
                curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &url2);
                fprintf(stderr, "R: %d - %s <%s>\n",
                        msg->data.result, curl_easy_strerror(msg->data.result), url2);
                curl_multi_remove_handle(multi_handle, e);
                curl_easy_cleanup(e);
                left--;
            } else {
                fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg);
            }
            if (transfers < HANDLECOUNT)
                add_transfer(multi_handle, transfers++, &left, file_names, file_path, url);
        }
        if (left)
            curl_multi_wait(multi_handle, NULL, 0, 1000, NULL);
    } while (left);
}

static size_t write_data(const char *ptr, size_t size, size_t nmemb, std::ofstream *userdata) {
    size_t nbytes = size * nmemb;
    userdata->write(ptr, nbytes);
    return nbytes;
}

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url) {
    CURL *eh = curl_easy_init();
    std::ofstream file_write((file_path + file_names[i]), std::ios::binary);
    std::cout << (file_path + file_names[i]) << " - " << (url + file_names[i]) << std::endl;
    curl_easy_setopt(eh, CURLOPT_URL, (url + file_names[i]).c_str());
    curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(eh, CURLOPT_WRITEDATA, &file_write);
    curl_multi_add_handle(cm, eh);
    (*left)++;
}

Answer 1

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url) {
    CURL *eh = curl_easy_init();
    std::ofstream file_write((file_path + file_names[i]), std::ios::binary);
    curl_easy_setopt(eh, CURLOPT_WRITEDATA, &file_write);
    // <...>
}

It stores the address of the local variable file_write that is destroyed at the function return. CURLOPT_WRITEDATA gets a dangling pointer.

static size_t write_data(const char *ptr, size_t size, size_t nmemb, std::ofstream *userdata) {
    size_t nbytes = size * nmemb;
    userdata->write(ptr, nbytes);
    return nbytes;
}

It uses the dangling pointer userdata, that results in undefined behavior. BTW the callback function signature is wrong in the last parameter, that also results in undefined behavior.

A possible solution is using the CURLOPT_PRIVATE curl easy option:

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url) {
    CURL *eh = curl_easy_init();
    auto *file_write = new std::ofstream ((file_path + file_names[i]), std::ios::binary);
    curl_easy_setopt(eh, CURLOPT_PRIVATE, file_write);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, file_write);
    // <...>
}

void download_files(std::vector<std::string> file_names, const std::string &file_path, const char *url,
                    int HANDLECOUNT) {
    // <...>
    do {
        // <...>
        while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
            if (msg->msg == CURLMSG_DONE) {
                // <...>
                curl_multi_remove_handle(multi_handle, e);
                void *file_write{};
                curl_easy_getinfo(e, CURLINFO_PRIVATE, &file_write);
                delete static_cast<std::ofstream*>(file_write);
                curl_easy_cleanup(e);
                left--;
            } else {
        // <...>
    } while (left);
}