OpenCV python is not displaying python wrapped Cuda SIFT Keypoints converted to OpenCV keypoint format

Question

We wrote a small python wrapper using pybind11 around the ExtractSift function which seems to work fine.

Here's what it looks like :

std::vector<SiftPoint> extractSIFT(char* filename)
{
    int devNum = 0, imgSet = 0;
    cv::Mat img;
    cv::imread(filename, 0).convertTo(img, CV_32FC1);
    unsigned int w = img.cols;
    unsigned int h = img.rows;

    InitCuda(devNum);
    CudaImage img1;
    img1.Allocate(w, h, iAlignUp(w, 128), false, NULL, (float*)img.data);
    img1.Download();
    
    SiftData siftData1;
    float initBlur = 1.0f;
    float thresh = (imgSet ? 4.5f : 3.0f);
    InitSiftData(siftData1, 32768, true, true); 
    float *memoryTmp = AllocSiftTempMemory(w, h, 5, false);
    ExtractSift(siftData1, img1, 5, initBlur, thresh, 0.0f, false, memoryTmp);
    FreeSiftTempMemory(memoryTmp);
    return std::vector<SiftPoint>(siftData1.h_data, siftData1.h_data + siftData1.numPts);
}

PYBIND11_MODULE(pycusift, m) {
    m.doc() = "SIFT feature extractor with CUDA"; // optional module docstring

    m.def("extractSIFT", &extractSIFT, "function to extract SIFT features from image");
    py::class_<SiftPoint>(m, "SiftPoint")
    .def(py::init<>())
    .def_readwrite("xpos", &SiftPoint::xpos)
    .def_readwrite("ypos", &SiftPoint::ypos)
    .def_readwrite("sharpness", &SiftPoint::sharpness)
    .def_readwrite("edgeness", &SiftPoint::edgeness)
    .def_readwrite("ambiguity", &SiftPoint::ambiguity)
    .def_readwrite("orientation", &SiftPoint::orientation)
    .def_readwrite("score", &SiftPoint::score)
    .def_readwrite("match", &SiftPoint::match)
    .def_readwrite("match_xpos", &SiftPoint::match_xpos)
    .def_readwrite("match_ypos", &SiftPoint::match_ypos)
    .def_readwrite("match_error", &SiftPoint::match_error)
    .def_readwrite("subsampling", &SiftPoint::ypos)
    .def_readwrite("scale", &SiftPoint::scale)
    .def("getempty", &SiftPoint::getempty)
    .def("getdata", &SiftPoint::getdata);
}

This type of usage correctly plots the keypoints

import pycusift

a = pycusift.extractSIFT("001.tiff")
x = [i.xpos for i in a]
y = [i.ypos for i in a]
import matplotlib.pyplot as plt
import cv2
plt.imshow(cv2.imread("001.tiff"))
plt.scatter(x,y)
plt.show() 

The following code attempts to display the keypoints using opencv but no keypoints get displayed, I reckon it's probably a float typecast issue

def extract_sift_keypoints(impath, kpts):
    im = cv.imread(impath, cv.IMREAD_COLOR)
    kp = cv.KeyPoint()
    kp1 = []
    desc1 = (np.array([k.getdata() for k in kpts], dtype=np.float32))
    

    for k in kpts :
        kp.pt = (k.xpos, k.ypos)
        kp.angle = k.orientation
        kp.size = k.scale
        kp1.append(kp)

    pts = np.array([k.pt for k in kp1], dtype=np.float32)
    ors = np.array([k.angle for k in kp1], dtype=np.float32)
    scs = np.array([k.size for k in kp1], dtype=np.float32) 

    return pts, ors, scs, desc1, im, kp1

if __name__ == '__main__':
    p = argparse.ArgumentParser()

    opt = p.parse_args()
    kpts1 = pycusift.extractSIFT(opt.im1)
    k1, o1, s1, d1, im1, kps1 = extract_sift_keypoints(opt.im1,kpts1)
   
    outimg = np.empty((im1.shape[0], im1.shape[1], 3), dtype=np.uint8)
    cv.drawKeypoints(im1, kps1, outimg)
    cv.imshow("kp", outimg)
    cv.waitKey(0)    

    print(type(kps1[0].pt))
    print(kps1[0].pt)
    print(type(kps1[0].angle))
    print(kps1[0].angle)
    bf = cv.BFMatcher()
    matches = bf.knnMatch(d1,d2, k=2)

I must point out that the knn descriptor matching I think is working as it's generating matches

Here is the output of the print statements :

<class 'tuple'>

(431.0779724121094, 982.3478393554688)

<class 'float'>

224.15390014648438

Answer 1

I found the solution, had to put kp = cv.keypoint() inside the for loop:

def extract_sift_keypoints(impath, kpts):
    im = cv.imread(impath, cv.IMREAD_COLOR)
    kp1 = []
    desc1 = (np.array([k.getdata() for k in kpts], dtype=np.float32))
    

    for k in kpts :
        kp = cv.KeyPoint()
        kp.pt = (k.xpos, k.ypos)
        kp.angle = k.orientation
        kp.size = k.scale
        kp1.append(kp)

    pts = np.array([k.pt for k in kp1], dtype=np.float32)
    ors = np.array([k.angle for k in kp1], dtype=np.float32)
    scs = np.array([k.size for k in kp1], dtype=np.float32) 

    return pts, ors, scs, desc1, im, kp1