opencv EstimateAffinePartial2d: What is the required input format?

Question

I'm attempting to find a "template" image in a "target" image and find a transform from template image coordinates to target image coordinates.

I've had a bit of success following this tutorial and using ORB to find keypoints, then a brute force matcher to find similar keypoints. Of the similar keypoints the code further filters good matches as per Lowe's ratio test. The remaining "good" keypoints are used with estimateAffinePartial2D to find a transform between the set of keypoints.

I've got a working version in python:

import sys
import numpy as np
import cv2.cv2 as cv2

# with the name image.jpg
img1 = cv2.imread('score_overlay_2021_1280.png')
img2 = cv2.imread('2021/frame-00570.jpg')

orb = cv2.ORB_create(nfeatures=1000) # Increasing nfeatures to get more keypoints

kp1, des1 = orb.detectAndCompute(img1, None)
kp2, des2 = orb.detectAndCompute(img2, None)


matcher = cv2.BFMatcher()
matches = matcher.knnMatch(des1, des2, k=2)

# Apply ratio test
good = []
for m, n in matches:
    if m.distance < 0.75 * n.distance:
        good.append(m)

print("found {} matches".format(len(good)))

if len(good) < 7:
    print("Not enough good keypoint matches between template and image")
    sys.exit(1)

src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
t = cv2.estimateAffinePartial2D(src_pts, dst_pts)

print(t)

I'm attempting to write the same code as above in go(lang) using the gocv bindings for opencv however I must be passing something incorrectly to estimateAffinePartial2D

Here is the error:

libc++abi.dylib: terminating with uncaught exception of type cv::Exception: OpenCV(4.5.3) /tmp/opencv-20210728-84579-13worgs/opencv-4.5.3/modules/calib3d/src/ptsetreg.cpp:1108: error: (-215:Assertion failed) count >= 0 && to.checkVector(2) == count in function 'estimateAffinePartial2D'

Here is the nearly identical go code that generates the above error

package main

import (
    "fmt"
    "log"
    "os"

    "gocv.io/x/gocv"
)

func run() error {
    var nfeatures = 1000                    // default is 500
    var scaleFactor float32 = 1.2           // default 1.2
    var nlevels = 8                         // default 8
    var edgeThreshold = 31                  // default 32
    var firstLevel = 0                      // default 0
    var WtaK = 2                            // default 2
    var scoreType = gocv.ORBScoreTypeHarris // default ORBScoreTypeHarris
    var patchSize = 31                      // default 31
    var fastThreshold = 20                  // default 20

    algo := gocv.NewORBWithParams(nfeatures, scaleFactor, nlevels, edgeThreshold, firstLevel, WtaK, scoreType, patchSize, fastThreshold)

    overlayFile := "score_overlay_2021_1280.png"
    overlay := gocv.IMRead(overlayFile, gocv.IMReadColor)
    if overlay.Empty() {
        return fmt.Errorf("unable to load %s", overlayFile)
    }
    kp1, des1 := algo.DetectAndCompute(overlay, gocv.NewMat())

    matcher := gocv.NewBFMatcher()

    frameFile := "frame-00570.jpg"
    img := gocv.IMRead(frameFile, gocv.IMReadColor)
    if img.Empty() {
        return fmt.Errorf("unable to load %s", frameFile)
    }
    kp2, des2 := algo.DetectAndCompute(img, gocv.NewMat())

    matches := matcher.KnnMatch(des1, des2, 4)

    // Store all the good matches as per Lowe's ratio test
    goodMatches := make([]gocv.DMatch, 0)
    for _, submatches := range matches {
        if submatches[0].Distance < 0.7*submatches[1].Distance {
            goodMatches = append(goodMatches, submatches[0])
        }
    }
    const MinGoodMatches = 6
    if len(goodMatches) < MinGoodMatches {
        return fmt.Errorf("only found %d matches, need at least %d", len(goodMatches), MinGoodMatches)
    }

    kp1vec := kp2Point2f(kp1)
    kp2vec := kp2Point2f(kp2)

    t := gocv.EstimateAffinePartial2D(kp1vec, kp2vec)

    log.Printf("T: %#v", t)
    return nil
}

func kp2Point2f(kp []gocv.KeyPoint) gocv.Point2fVector {
    kp2f := make([]gocv.Point2f, 0)
    for _, kp := range kp {
        kp2f = append(kp2f, gocv.Point2f{
            X: float32(kp.X),
            Y: float32(kp.Y),
        })
    }
    return gocv.NewPoint2fVectorFromPoints(kp2f)
}

func main() {
    if err := run(); err != nil {
        log.Printf("Error: %s", err)
        os.Exit(1)
    }
}

Template Image: https://github.com/TechplexEngineer/frc-livescore/blob/orb-test/orbtest/score_overlay_2021_1280.png

Frame: https://github.com/TechplexEngineer/frc-livescore/blob/orb-test/orbtest/2021/frame-00570.jpg

Answer 1

Turns out the go(lang) code above is not applying Lowe's ratio test properly.

Here is working code:

package main

import (
    "fmt"
    "image"
    "log"
    "os"

    "gocv.io/x/gocv"
)

func run() error {
    var nfeatures = 1000                    // default is 500
    var scaleFactor float32 = 1.2           // default 1.2
    var nlevels = 8                         // default 8
    var edgeThreshold = 31                  // default 32
    var firstLevel = 0                      // default 0
    var WtaK = 2                            // default 2
    var scoreType = gocv.ORBScoreTypeHarris // default ORBScoreTypeHarris
    var patchSize = 31                      // default 31
    var fastThreshold = 20                  // default 20

    algo := gocv.NewORBWithParams(nfeatures, scaleFactor, nlevels, edgeThreshold, firstLevel, WtaK, scoreType, patchSize, fastThreshold)

    overlayFile := "score_overlay_2021_1280.png"
    overlay := gocv.IMRead(overlayFile, gocv.IMReadColor)
    if overlay.Empty() {
        return fmt.Errorf("unable to load %s", overlayFile)
    }
    kp1, des1 := algo.DetectAndCompute(overlay, gocv.NewMat())

    matcher := gocv.NewBFMatcher()

    frameFile := "frame-00570.jpg"
    img := gocv.IMRead(frameFile, gocv.IMReadColor)
    if img.Empty() {
        return fmt.Errorf("unable to load %s", frameFile)
    }

    newWidth := 1280.0
    newHeight := newWidth*(float64(img.Rows())/float64(img.Cols()))
    gocv.Resize(img, &img, image.Pt(int(newWidth), int(newHeight)), 0,0, gocv.InterpolationDefault)

    kp2, des2 := algo.DetectAndCompute(img, gocv.NewMat())

    matches := matcher.KnnMatch(des1, des2, 4)

    // Store all the good matches as per Lowe's ratio test
    goodMatches := make([]gocv.DMatch, 0)
    for _, submatches := range matches {
        if submatches[0].Distance < 0.75 * submatches[1].Distance {
            goodMatches = append(goodMatches, submatches[0])
        }
    }
    const MinGoodMatches = 7
    if len(goodMatches) < MinGoodMatches {
        return fmt.Errorf("only found %d matches, need at least %d", len(goodMatches), MinGoodMatches)
    }

    kp1f := make([]gocv.Point2f, 0)
    for _, gm := range goodMatches {
        kp1f = append(kp1f, gocv.Point2f{
            X: float32(kp1[gm.QueryIdx].X),
            Y: float32(kp1[gm.QueryIdx].Y),
        })
    }
    kp1vec := gocv.NewPoint2fVectorFromPoints(kp1f)

    kp2f := make([]gocv.Point2f, 0)
    for _, gm := range goodMatches {
        kp2f = append(kp2f, gocv.Point2f{
            X: float32(kp2[gm.TrainIdx].X),
            Y: float32(kp2[gm.TrainIdx].Y),
        })
    }
    kp2vec := gocv.NewPoint2fVectorFromPoints(kp2f)

    log.Printf("kp1:%d kp2:%d", kp1vec.Size(),kp2vec.Size())

    t := gocv.EstimateAffinePartial2D(kp1vec, kp2vec)
    
    scale := t.GetDoubleAt(0,0)
    tx := t.GetDoubleAt(0,2)
    ty := t.GetDoubleAt(1,2)

    log.Printf("Scale: %f Tx: %f Ty: %f", scale, tx, ty)
    return nil
}

func main() {
    if err := run(); err != nil {
        log.Printf("Error: %s", err)
        os.Exit(1)
    }
}