Map screen coordinates to texture coordinates of a face mesh using ARKit

Question

I'm trying to make an AR app using ARkit that allows the user to draw lipstick on their faces. How do I map the screen coordinates to the texture coordinates for the face mesh?

func transformToTextureCoordinates(screenCoordinates: CGPoint) -> CGPoint {

     // hit test from screen to the face geometry
     let hitTestResults = sceneView.hitTest(screenCoordinates, options: nil)
     guard let result = hitTestResults.first else {
         return CGPoint(x: -1.0, y: -1.0)
     let world_coords = result.worldCoordinates

     --- HELP! ---
}

func drawLine(from fromPoint: CGPoint, to toPoint: CGPoint) {
      // transform the screen coordinates to texture coordinates
      let fromPoint_transformed = transformToTextureCoordinates(screenCoordinates: fromPoint)
      let toPoint_transformed = transformToTextureCoordinates(screenCoordinates:toPoint)

      // draw line on the texture image
      UIGraphicsBeginImageContext(view.frame.size)
          guard let context = UIGraphicsGetCurrentContext() else {
          return
      }
      textureImage?.draw(in: view.bounds)
      context.move(to: fromPoint_transformed)
      context.addLine(to: toPoint_transformed)
      context.setLineCap(.round)
      context.setBlendMode(.normal)
      context.setLineWidth(brushWidth)
      context.setStrokeColor(color.cgColor)
      context.strokePath()
      textureImage = UIGraphicsGetImageFromCurrentImageContext()
      UIGraphicsEndImageContext()
}

Answer 1

It's actually super simple. No need for transformation matrix. Hit test provides the texture coordinates u,v in the 0 to 1 range. So multiply by the texture width and height, you will get the pixel coordinates on the texture

func transformToTextureCoordinates(screenCoordinates: CGPoint) -> CGPoint {
    let hitTestResults = sceneView.hitTest(screenCoordinates, options: nil)
    guard let result = hitTestResults.first else {
        return CGPoint(x: -1, y: -1)
    }
    let textureCoordinates = result.textureCoordinates(withMappingChannel: 0)
    return CGPoint(x: textureCoordinates.x * textureImage.size.width, y: textureCoordinates.y * textureImage.size.height)
}