Upscaling with Realesrgan

Question

I am trying to create a cpp file to use the RealESRGAN_x4plus_anime model to upscale an image.

I did the following:

#include <iostream>
#include <string>
#include <net.h>
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>

void upscaleWithRealESRGAN(const std::string& inputImagePath, const std::string& outputImagePath, int width, int height) {
    // Initialize ncnn net
    ncnn::Net realesrgan;

    // Load the pre-trained RealESRGAN model files
    realesrgan.load_param("assets/RealESRGAN_x4plus_anime.param");
    realesrgan.load_model("assets/RealESRGAN_x4plus_anime.bin");

    // Load the input image using OpenCV
    cv::Mat img = cv::imread(inputImagePath, cv::IMREAD_COLOR);
    if (img.empty()) {
        std::cerr << "Error loading image from path: " << inputImagePath << std::endl;
        return;
    }

    // Convert cv::Mat to ncnn::Mat
    ncnn::Mat in = ncnn::Mat::from_pixels(img.data, ncnn::Mat::PIXEL_BGR2RGB, img.cols, img.rows);

    if (width != 0 && height != 0) {
        ncnn::resize_bilinear(in, in, width, height);
    }

    // Create an extractor for inference
    ncnn::Extractor ex = realesrgan.create_extractor();

    // Input the image to the network
    ex.input("data", in);

    // Create an ncnn::Mat object for the output
    ncnn::Mat out;

    // Run the forward pass to get the upscaled image
    ex.extract("output", out);

    if (out.empty()) {
        std::cerr << "Neural network output is empty." << std::endl;
        return;
    }

    // Convert the output ncnn::Mat back to cv::Mat for saving
    cv::Mat out_cv(img.rows * 4, img.cols * 4, CV_8UC3); // Assuming x4 upscaling
    out.to_pixels(out_cv.data, ncnn::Mat::PIXEL_RGB2BGR);
    
    if (out_cv.empty()) {
        std::cerr << "Conversion to cv::Mat resulted in an empty matrix." << std::endl;
        return;
    }

    // Save the upscaled image using OpenCV
    cv::imwrite(outputImagePath, out_cv);
}

int main(int argc, char* argv[]) {
    std::string inputPath = "input.jpg";  // Default input path
    std::string outputPath = "./output.png";  // Default output path
    int width = 0;  // Initialize with default 0
    int height = 0;  // Initialize with default 0

    for (int i = 1; i < argc; i++) {
        if (std::string(argv[i]) == "-input" && i + 1 < argc) {
            inputPath = argv[i + 1];
            i++;  // skip next arg
        } else if (std::string(argv[i]) == "-output" && i + 1 < argc) {
            outputPath = argv[i + 1];
            i++;  // skip next arg
        } else if (std::string(argv[i]) == "-w" && i + 1 < argc) {
            width = std::stoi(argv[i + 1]);
            i++;  // skip next arg
        } else if (std::string(argv[i]) == "-h" && i + 1 < argc) {
            height = std::stoi(argv[i + 1]);
            i++;  // skip next arg
        }
    }

    upscaleWithRealESRGAN(inputPath, outputPath, width, height);
    std::cout << "Upscaling completed!" << std::endl;
    return 0;
}

I get this error:

marcus@adming-VirtualBox:~/Desktop/Code/_stable-diffusion-cpp/linux$ ./build/upscaler-ncnn -input result_15_42_256x256.png -w 512 -h 512
Error loading image!
Upscaling completed!

The image is in the folder as the upscaler-ncnn, however it does not execute.

Any suggestions what I am doing wrong?

I appreciate your replies!

Answer 1

your current working directory is set to the parent of build, the app is looking for the image file there, you need to either

1. move the image to the current working directory (the parent of build)
2. provide image path relative to current working directory ./build/result_15_42_256x256.png
3. cd to the build directory, cd build