Aspect fill AVCaptureVideoDataOutput when drawing in GLKView with CIContext

Question

I'm drawing a camera output from AVCaptureVideoDataOutput in a GLKView, but the camera is 4:3, which doesn't match the aspect ratio of the GLKView (which is full screen). I'm trying to get an aspect fill, but the camera output just seems to get squashed so that it doesn't go over the edge the frame of the view. How can I get a full screen camera view using GLKView without messing up the aspect ratio?

Initialising the view:

videoDisplayView = GLKView(frame: superview.bounds, context: EAGLContext(api: .openGLES2))
videoDisplayView.transform = CGAffineTransform(rotationAngle: CGFloat(M_PI_2))
videoDisplayView.frame = superview.bounds
superview.addSubview(videoDisplayView)
superview.sendSubview(toBack: videoDisplayView)

renderContext = CIContext(eaglContext: videoDisplayView.context)
sessionQueue = DispatchQueue(label: "AVSessionQueue", attributes: [])

videoDisplayView.bindDrawable()
videoDisplayViewBounds = CGRect(x: 0, y: 0, width: videoDisplayView.drawableWidth, height: videoDisplayView.drawableHeight)

Initialising the video output:

let videoOutput = AVCaptureVideoDataOutput()
videoOutput.setSampleBufferDelegate(self, queue: sessionQueue)
if captureSession.canAddOutput(videoOutput) {
    captureSession.addOutput(videoOutput)
}

Rendering the output:

func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {

    // Need to shimmy this through type-hell
    let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
    // Force the type change - pass through opaque buffer
    let opaqueBuffer = Unmanaged<CVImageBuffer>.passUnretained(imageBuffer!).toOpaque()
    let pixelBuffer = Unmanaged<CVPixelBuffer>.fromOpaque(opaqueBuffer).takeUnretainedValue()

    let sourceImage = CIImage(cvPixelBuffer: pixelBuffer, options: nil)

    // Do some detection on the image
    let detectionResult = applyFilter?(sourceImage)
    var outputImage = sourceImage
    if detectionResult != nil {
        outputImage = detectionResult!
    }

    if videoDisplayView.context != EAGLContext.current() {
        EAGLContext.setCurrent(videoDisplayView.context)
    }
    videoDisplayView.bindDrawable()

    // clear eagl view to grey
    glClearColor(0.5, 0.5, 0.5, 1.0);
    glClear(0x00004000)

    // set the blend mode to "source over" so that CI will use that
    glEnable(0x0BE2);
    glBlendFunc(1, 0x0303);

    renderContext.draw(outputImage, in: videoDisplayViewBounds, from: outputImage.extent)

    videoDisplayView.display()
}

Things I've tried:

// Results in 4:3 stream leaving a gap at the bottom
renderContext.draw(outputImage, in: outputImage.extent, from: outputImage.extent)

// Results in same 4:3 stream
let rect = CGRect(x: 0, y: 0, width: outputImage.extent.width, height: videoDisplayViewBounds.height)
renderContext.draw(outputImage, in: rect, from: outputImage.extent)

Answer 1

I actually ended up having to crop my output to the size of the view I was displaying the output in.

func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {

    // Need to shimmy this through type-hell
    let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
    // Force the type change - pass through opaque buffer
    let opaqueBuffer = Unmanaged<CVImageBuffer>.passUnretained(imageBuffer!).toOpaque()
    let pixelBuffer = Unmanaged<CVPixelBuffer>.fromOpaque(opaqueBuffer).takeUnretainedValue()

    let sourceImage = CIImage(cvPixelBuffer: pixelBuffer, options: nil)

    // Make a rect to crop to that's the size of the view we want to display the image in
    let cropRect = AVMakeRect(aspectRatio: CGSize(width: videoDisplayViewBounds.width, height: videoDisplayViewBounds.height), insideRect: sourceImage.extent)
    // Crop
    let croppedImage = sourceImage.cropping(to: cropRect)
    // Cropping changes the origin coordinates of the cropped image, so move it back to 0
    let translatedImage = croppedImage.applying(CGAffineTransform(translationX: 0, y: -croppedImage.extent.origin.y))

    // Do some detection on the image
    let detectionResult = applyFilter?(translatedImage)
    var outputImage = translatedImage
    if detectionResult != nil {
        outputImage = detectionResult!
    }

    if videoDisplayView.context != EAGLContext.current() {
        EAGLContext.setCurrent(videoDisplayView.context)
    }
    videoDisplayView.bindDrawable()

    // clear eagl view to grey
    glClearColor(0.5, 0.5, 0.5, 1.0)
    glClear(0x00004000)

    // set the blend mode to "source over" so that CI will use that
    glEnable(0x0BE2);
    glBlendFunc(1, 0x0303)

    renderContext.draw(outputImage, in: videoDisplayViewBounds, from: outputImage.extent)

    videoDisplayView.display()
}