MTKView Drawing Performance

Question

What I am Trying to Do

I am trying to show filters on a camera feed by using a Metal view: MTKView. I am closely following the method of Apple's sample code - Enhancing Live Video by Leveraging TrueDepth Camera Data (link).

What I Have So Far

Following code works great (mainly interpreted from above-mentioned sample code) :

    class MetalObject: NSObject, MTKViewDelegate {

            private var metalBufferView         : MTKView?
            private var metalDevice             = MTLCreateSystemDefaultDevice()
            private var metalCommandQueue       : MTLCommandQueue!

            private var ciContext               : CIContext!
            private let colorSpace              = CGColorSpaceCreateDeviceRGB()

            private var videoPixelBuffer        : CVPixelBuffer?

            private let syncQueue               = DispatchQueue(label: "Preview View Sync Queue", qos: .userInitiated, attributes: [], autoreleaseFrequency: .workItem)

            private var textureWidth            : Int             = 0
            private var textureHeight           : Int             = 0
            private var textureMirroring        = false
            private var sampler                 : MTLSamplerState!
            private var renderPipelineState     : MTLRenderPipelineState!
            private var vertexCoordBuffer       : MTLBuffer!
            private var textCoordBuffer         : MTLBuffer!
            private var internalBounds          : CGRect!
            private var textureTranform         : CGAffineTransform?

            private var previewImage            : CIImage?

    init(with frame: CGRect) {
        super.init()

        self.metalBufferView = MTKView(frame: frame, device: self.metalDevice)
        self.metalBufferView!.contentScaleFactor = UIScreen.main.nativeScale
        self.metalBufferView!.framebufferOnly = true
        self.metalBufferView!.colorPixelFormat = .bgra8Unorm
        self.metalBufferView!.isPaused = true
        self.metalBufferView!.enableSetNeedsDisplay = false
        self.metalBufferView!.delegate = self

        self.metalCommandQueue = self.metalDevice!.makeCommandQueue()

        self.ciContext = CIContext(mtlDevice: self.metalDevice!)


        //Configure Metal
        let defaultLibrary = self.metalDevice!.makeDefaultLibrary()!
        let pipelineDescriptor = MTLRenderPipelineDescriptor()
        pipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm
        pipelineDescriptor.vertexFunction = defaultLibrary.makeFunction(name: "vertexPassThrough")
        pipelineDescriptor.fragmentFunction = defaultLibrary.makeFunction(name: "fragmentPassThrough")

        // To determine how our textures are sampled, we create a sampler descriptor, which
        // will be used to ask for a sampler state object from our device below.
        let samplerDescriptor = MTLSamplerDescriptor()
        samplerDescriptor.sAddressMode = .clampToEdge
        samplerDescriptor.tAddressMode = .clampToEdge
        samplerDescriptor.minFilter = .linear
        samplerDescriptor.magFilter = .linear

        sampler = self.metalDevice!.makeSamplerState(descriptor: samplerDescriptor)

        do {
            renderPipelineState = try self.metalDevice!.makeRenderPipelineState(descriptor: pipelineDescriptor)
        } catch {
            fatalError("Unable to create preview Metal view pipeline state. (\(error))")
        }

    }




    final func update (newVideoPixelBuffer: CVPixelBuffer?) {

        self.syncQueue.async {

            var filteredImage : CIImage

            self.videoPixelBuffer = newVideoPixelBuffer

            //---------
            //Core image filters
            //Strictly CIFilters, chained together
            //---------

            self.previewImage = filteredImage

            //Ask Metal View to draw
            self.metalBufferView?.draw()

        }
    }



    //MARK: - Metal View Delegate
    final func draw(in view: MTKView) {

        print (Thread.current)

        guard let drawable = self.metalBufferView!.currentDrawable,
            let currentRenderPassDescriptor = self.metalBufferView!.currentRenderPassDescriptor,
            let previewImage = self.previewImage else {
                return
        }


        // create a texture for the CI image to render to
        let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(
            pixelFormat: .bgra8Unorm,
            width: Int(previewImage.extent.width),
            height: Int(previewImage.extent.height),
            mipmapped: false)
        textureDescriptor.usage = [.shaderWrite, .shaderRead]

        let texture = self.metalDevice!.makeTexture(descriptor: textureDescriptor)!

        if texture.width != textureWidth ||
            texture.height != textureHeight ||
            self.metalBufferView!.bounds != internalBounds {
            setupTransform(width: texture.width, height: texture.height, mirroring: mirroring, rotation: rotation)
        }

        // Set up command buffer and encoder
        guard let commandQueue = self.metalCommandQueue else {
            print("Failed to create Metal command queue")
            return
        }

        guard let commandBuffer = commandQueue.makeCommandBuffer() else {
            print("Failed to create Metal command buffer")
            return
        }

        // add rendering of the image to the command buffer
        ciContext.render(previewImage,
                         to: texture,
                         commandBuffer: commandBuffer,
                         bounds: previewImage.extent,
                         colorSpace: self.colorSpace)

        guard let commandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor) else {
            print("Failed to create Metal command encoder")
            return
        }

        // add vertex and fragment shaders to the command buffer
        commandEncoder.label = "Preview display"
        commandEncoder.setRenderPipelineState(renderPipelineState!)
        commandEncoder.setVertexBuffer(vertexCoordBuffer, offset: 0, index: 0)
        commandEncoder.setVertexBuffer(textCoordBuffer, offset: 0, index: 1)
        commandEncoder.setFragmentTexture(texture, index: 0)
        commandEncoder.setFragmentSamplerState(sampler, index: 0)
        commandEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
        commandEncoder.endEncoding()

        commandBuffer.present(drawable) // Draw to the screen
        commandBuffer.commit()

    }


    final func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {

    }

}

Notes

• The reason MTKViewDelegate is used instead of subclassing MTKView is that when it was subclassed, the draw call was called on the main thread. With the delegate method shown above, it seems to be a different metal related thread call each loop. Above method seem to give much better performance.
• Details on CIFilter usage on update method above had to be redacted. All it is a heavy chain of CIFilters stacked. Unfortunately there is no room for any tweaks with these filters.

Issue

Above code seems to slow down the main thread a lot, causing rest of the app UI to be choppy. For example, scrolling a UIScrollview gets seem to be slow and choppy.

Goal

Tweak Metal view to ease up on CPU and go easy on the main thread to leave enough juice for rest of the UI.

According to the above graphics, preparation of command buffer is all done in CPU until presented and committed(?). Is there a way to offload that from CPU?

Any hints, feedback, tips, etc to improve the drawing efficiency would be appreciated.

Answer 1

There are a few things you can do to improve the performance:

• Render into the view’s drawable directly instead of rendering into a texture and then rendering again to render that texture into the view.
• Use the newish CIRenderDestination API to defer the actual texture retrieval to the moment the view is actually rendered to (i.e. when Core Image is done).

Here’s the draw(in view: MTKView) I’m using in my Core Image project, modified for your case:

public func draw(in view: MTKView) {
    if let currentDrawable = view.currentDrawable,
        let commandBuffer = self.commandQueue.makeCommandBuffer() {
        let drawableSize = view.drawableSize

        // optional: scale the image to fit the view
        let scaleX = drawableSize.width / image.extent.width
        let scaleY = drawableSize.height / image.extent.height
        let scale = min(scaleX, scaleY)
        let scaledImage = previewImage.transformed(by: CGAffineTransform(scaleX: scale, y: scale))

        // optional: center in the view
        let originX = max(drawableSize.width - scaledImage.extent.size.width, 0) / 2
        let originY = max(drawableSize.height - scaledImage.extent.size.height, 0) / 2
        let centeredImage = scaledImage.transformed(by: CGAffineTransform(translationX: originX, y: originY))

        // create a render destination that allows to lazily fetch the target texture
        // which allows the encoder to process all CI commands _before_ the texture is actually available;
        // this gives a nice speed boost because the CPU doesn’t need to wait for the GPU to finish
        // before starting to encode the next frame
        let destination = CIRenderDestination(width: Int(drawableSize.width),
                                              height: Int(drawableSize.height),
                                              pixelFormat: view.colorPixelFormat,
                                              commandBuffer: commandBuffer,
                                              mtlTextureProvider: { () -> MTLTexture in
                                                return currentDrawable.texture
        })

        let task = try! self.context.startTask(toRender: centeredImage, to: destination)
        // bonus: you can Quick Look the task to see what’s actually scheduled for the GPU

        commandBuffer.present(currentDrawable)
        commandBuffer.commit()

        // optional: you can wait for the task execution and Quick Look the info object to get insights and metrics
        DispatchQueue.global(qos: .background).async {
            let info = try! task.waitUntilCompleted()
        }
    }
}

If this is still too slow, you can try setting the priorityRequestLow CIContextOption when creating your CIContext to tell Core Image to render in low priority.