OpenCV match image from camera with same image does not produce 100% matching

Question

My goal is to match an image captured from a camera with some models and find the closest one. However I think I am missing something. This is what I'm doing: first I get a frame from the camera, select a portion, extract keypoints and compute descriptors using SURF and store them in a xml file (I also store the model as model.png). This is my model. Then I take another frame (in few seconds), select the same portion, compute descriptors and match these against the previously stored one. The result is not close to 100% (I use the ratio between good matches and number of keypoints) like I would expect. To have a comparison, if I load model.png, compute its descriptors and match against the stored descriptors I get 100% matching (more or less), and this is reasonable. This is my code:

#include <iostream>
#include "opencv2/opencv.hpp"
#include "opencv2/nonfree/nonfree.hpp"

using namespace std;

std::vector<cv::KeyPoint> detectKeypoints(cv::Mat image, int hessianTh, int nOctaves, int nOctaveLayers, bool extended, bool upright) {
    std::vector<cv::KeyPoint> keypoints;
    cv::SurfFeatureDetector detector(hessianTh,nOctaves,nOctaveLayers,extended,upright);
    detector.detect(image,keypoints);
    return keypoints; }

cv::Mat computeDescriptors(cv::Mat image,std::vector<cv::KeyPoint> keypoints, int hessianTh, int nOctaves, int nOctaveLayers, bool extended, bool upright) {
    cv::SurfDescriptorExtractor extractor(hessianTh,nOctaves,nOctaveLayers,extended,upright);
    cv::Mat imageDescriptors;
    extractor.compute(image,keypoints,imageDescriptors);
    return imageDescriptors; }

int main(int argc, char *argv[]) {
    cv::VideoCapture cap(0);
    cap.set(CV_CAP_PROP_FRAME_WIDTH, 2304); 
    cap.set(CV_CAP_PROP_FRAME_HEIGHT, 1536); 
    cap >> frame;
    cv::Rect selection(939,482,1063-939,640-482);

    cv::Mat roi = frame(selection).clone();
    //cv::Mat roi=cv::imread("model.png");  
    cv::cvtColor(roi,roi,CV_BGR2GRAY);
    cv::equalizeHist(roi,roi);

    if (std::stoi(argv[1])==1)
    {
        std::vector<cv::KeyPoint> keypoints = detectKeypoints(roi,400,4,2,true,false);
        cv::FileStorage fs("model.xml", cv::FileStorage::WRITE);
        cv::write(fs,"keypoints",keypoints);
        cv::write(fs,"descriptors",computeDescriptors(roi,keypoints,400,4,2,true,false));
        fs.release();
        cv::imwrite("model.png",roi);
    }
    else
    {
        cv::FileStorage fs("model.xml", cv::FileStorage::READ);
        std::vector<cv::KeyPoint> modelkeypoints;
        cv::Mat modeldescriptor;
        cv::FileNode filenode = fs["keypoints"];
        cv::read(filenode,modelkeypoints);
        filenode = fs["descriptors"];
        cv::read(filenode, modeldescriptor);
        fs.release();

        std::vector<cv::KeyPoint> roikeypoints = detectKeypoints(roi,400,4,2,true,false);
        cv::Mat roidescriptor = computeDescriptors(roi,roikeypoints,400,4,2,true,false);

        std::vector<std::vector<cv::DMatch>> matches;
        cv::BFMatcher matcher(cv::NORM_L2);
        if(roikeypoints.size()<modelkeypoints.size())
            matcher.knnMatch(roidescriptor, modeldescriptor, matches, 2);  // Find two nearest matches
        else
            matcher.knnMatch(modeldescriptor, roidescriptor, matches, 2);

        vector<cv::DMatch> good_matches;
        for (int i = 0; i < matches.size(); ++i)
        {
            const float ratio = 0.7;
            if (matches[i][0].distance < ratio * matches[i][1].distance)
            {
                good_matches.push_back(matches[i][0]);
            }
        }

        cv::Mat matching;

        cv::Mat model = cv::imread("model.png");
        if(roikeypoints.size()<modelkeypoints.size())
            cv::drawMatches(roi,roikeypoints,model,modelkeypoints,good_matches,matching);
        else
            cv::drawMatches(model,modelkeypoints,roi,roikeypoints,good_matches,matching);

        cv::imwrite("matches.png",matching);

        float result = static_cast<float>(good_matches.size())/static_cast<float>(roikeypoints.size());
        std::cout << result << std::endl;
    }
    return 0; }

Any suggestion will be appreciated, this is driving me crazy..

Answer 1

This is expected, the small change between the two frames is the reason you don't get 100% matches. But on the same image, the SURF features are going to be exactly at the same points and the computed descriptors are going to be identical. So tune your method for your camera, plot the distance between features when they are supposed to be identical. Set a threshold on the distance such that most (maybe 95%) of the matches are accepted. This way you will have a low false match rate and still have a large rate of true matches.