Stitching Video with fast playback of frames

I'm attempting to stitch two videos together though matching there key points though finding the homography between the overlapping video. I have successfully got this to work with two different images.

With the video I have loaded the two separate video files and looped the frames and copied them to the blank matrix cap1frame and cap2frame for each video.

Then I send each frame from each video to the stitching function which matches the keypoints based on the homography between the two frames and stitch them and display the resultant image. (matching based on openCV example)

The stitching is successful however, it results in a very slow playback of the video and some sort of graphical anomalies on the side of the frame. Seen in the photo.

I'm wondering how I can make this more efficient with fast video playback.

int main(int argc, char** argv){
      // Create a VideoCapture object and open the input file
      VideoCapture cap1("left.mov");
      VideoCapture cap2("right.mov");
      // Check if camera opened successfully
      if(!cap1.isOpened() || !cap2.isOpened()){
        cout << "Error opening video stream or file" << endl;
        return -1;
      }
        //Trying to loop frames
        for (;;){
        Mat cap1frame;
        Mat cap2frame;

        cap1 >> cap1frame;
        cap2 >> cap2frame;

        // If the frame is empty, break immediately
        if (cap1frame.empty() || cap2frame.empty())
          break;

        //sending each frame from each video to the stitch function then displaying
        imshow( "Result", Stitching(cap1frame,cap2frame));

        if(waitKey(30) >= 0) break;
         //destroyWindow("Stitching");
        // waitKey(0);
      }
      return 0;
    }

Solution

I was able to resolve my issue by pre-calculating the homography with just the first frame of video. This is so the function was only called once.

I then looped through the rest of the video to apply the warping of the video frames so they could be stitched together based on the pre-calculated homography. This bit was initially within my stitching function.

I still had an issue at this point with playback still being really slow when calling imshow. But I decided to export the resultant video and this worked when the correct fps was set in the VideoWriter object. I wonder if I just needed to adjust the fps playback of imshow but I'm not sure on that bit.

I've got my full code below:

#include <stdio.h>
#include <iostream>
#include "opencv2/core.hpp"
#include "opencv2/features2d.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/calib3d.hpp"
#include "opencv2/xfeatures2d.hpp"
#include <opencv2/xfeatures2d/nonfree.hpp>
#include <opencv2/xfeatures2d/cuda.hpp>
#include <opencv2/opencv.hpp>
#include <vector>
//To get homography from images passed in. Matching points in the images.

Mat Stitching(Mat image1,Mat image2){

    Mat I_1 = image1;
    Mat I_2 = image2;
//based on https://docs.opencv.org/3.3.0/d7/dff/tutorial_feature_homography.html
    cv::Ptr<Feature2D> f2d = xfeatures2d::SIFT::create();
        // Step 1: Detect the keypoints:
        std::vector<KeyPoint> keypoints_1, keypoints_2;
        f2d->detect( I_1, keypoints_1 );
        f2d->detect( I_2, keypoints_2 );
        // Step 2: Calculate descriptors (feature vectors)
        Mat descriptors_1, descriptors_2;
        f2d->compute( I_1, keypoints_1, descriptors_1 );
        f2d->compute( I_2, keypoints_2, descriptors_2 );
        // Step 3: Matching descriptor vectors using BFMatcher :
        BFMatcher matcher;
        std::vector< DMatch > matches;
        matcher.match( descriptors_1, descriptors_2, matches );
        // Keep best matches only to have a nice drawing.
        // We sort distance between descriptor matches
        Mat index;
        int nbMatch = int(matches.size());
        Mat tab(nbMatch, 1, CV_32F);
        for (int i = 0; i < nbMatch; i++)
            tab.at<float>(i, 0) = matches[i].distance;
        sortIdx(tab, index, SORT_EVERY_COLUMN + SORT_ASCENDING);
        vector<DMatch> bestMatches;
        for (int i = 0; i < 200; i++)
            bestMatches.push_back(matches[index.at < int > (i, 0)]);
        // 1st image is the destination image and the 2nd image is the src image
        std::vector<Point2f> dst_pts;                   //1st
        std::vector<Point2f> source_pts;                //2nd

        for (vector<DMatch>::iterator it = bestMatches.begin(); it != bestMatches.end(); ++it) {
            //cout << it->queryIdx << "\t" <<  it->trainIdx << "\t"  <<  it->distance << "\n";
            //-- Get the keypoints from the good matches
            dst_pts.push_back( keypoints_1[ it->queryIdx ].pt );
            source_pts.push_back( keypoints_2[ it->trainIdx ].pt );
        }
        Mat H_12 = findHomography( source_pts, dst_pts, CV_RANSAC );
      return H_12;
}
int main(int argc, char** argv){
  //Mats to get the first frame of video and pass to Stitching function.
  Mat I1, h_I1;
  Mat I2, h_I2;
  // Create a VideoCapture object and open the input file
  VideoCapture cap1("left.mov");
  VideoCapture cap2("right.mov");
  cap1.set(CV_CAP_PROP_BUFFERSIZE, 10);
  cap2.set(CV_CAP_PROP_BUFFERSIZE, 10);
  //Check if camera opened successfully
  if(!cap1.isOpened() || !cap2.isOpened()){
    cout << "Error opening video stream or file" << endl;
    return -1;
  }
//passing first frame to Stitching function
  if (cap1.read(I1)){
     h_I1 = I1;
   }

   if (cap2.read(I2)){
     h_I2 = I2;
   }
   Mat homography;
//passing here.
   homography = Stitching(h_I1,h_I2);
  std::cout << homography << '\n';

//creating VideoWriter object with defined values.
VideoWriter video("video/output.avi",CV_FOURCC('M','J','P','G'),30, Size(1280,720));

//Looping through frames of both videos.
    for (;;){
    Mat cap1frame;
    Mat cap2frame;

    cap1 >> cap1frame;
    cap2 >> cap2frame;

    // If the frame is empty, break immediately
    if (cap1frame.empty() || cap2frame.empty())
      break;
      Mat warpImage2;
      //warping the second video cap2frame so it matches with the first one.
      //size is defined as the final video size
      warpPerspective(cap2frame, warpImage2, homography, Size(1280,720), INTER_CUBIC);
      //final is the final canvas where both videos will be warped onto.
      Mat final (Size(1280,720), CV_8UC3);
      //Mat final(Size(cap1frame.cols*2 + cap1frame.cols, cap1frame.rows*2),CV_8UC3);
      //Using roi getting the relivent areas of each video.
      Mat roi1(final, Rect(0, 0,  cap1frame.cols, cap1frame.rows));
      Mat roi2(final, Rect(0, 0, warpImage2.cols, warpImage2.rows));
      //warping images on to the canvases which are linked with the final canvas.
      warpImage2.copyTo(roi2);
      cap1frame.copyTo(roi1);
      //writing to video.
      video.write(final);
      //imshow ("Result", final);
    if(waitKey(30) >= 0) break;
  }
  video.release();
  return 0;
}