来自 ARKit 相机框架的纹理 ARMeshGeometry？

Question

这个问题有点建立在这篇文章的基础上，其中的想法是ARMeshGeometry从带有 LiDAR 扫描仪的 iOS 设备中获取数据，计算纹理坐标，并将采样的相机帧应用为给定网格的纹理，从而允许用户创建一个“逼真的”3D 表示他们的环境。

根据那篇文章，我已经调整了其中一个响应来计算纹理坐标，如下所示；

func buildGeometry(meshAnchor: ARMeshAnchor, arFrame: ARFrame) -> SCNGeometry {
    let vertices = meshAnchor.geometry.vertices

    let faces = meshAnchor.geometry.faces
    let camera = arFrame.camera
    let size = arFrame.camera.imageResolution
    
    // use the MTL buffer that ARKit gives us
    let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat: vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride)
    
    // set the camera matrix
    let modelMatrix = meshAnchor.transform
    
    var textCords = [CGPoint]()
    for index in 0..<vertices.count {
        let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
        let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
        let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1)
        let world_vertex4 = simd_mul(modelMatrix, vertex4)
        let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
        let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width)))
        let v = 1.0 - Float(pt.x) / Float(size.height)
        let u = Float(pt.y) / Float(size.width)
        
        //let z = vector_float2(u, v)
        let c = CGPoint(x: v, y: u)
        textCords.append(c)
    }
    
    // Setup the texture coordinates
    let textureSource = SCNGeometrySource(textureCoordinates: textCords)
    
    // Setup the normals
    let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal)
    
    // Setup the geometry
    let faceData = Data(bytesNoCopy: faces.buffer.contents(), count: faces.buffer.length, deallocator: .none)
    let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
    let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement])
    
    /* Setup texture - THIS IS WHERE I AM STUCK
    let texture = textureConverter.makeTextureForMeshModel(frame: arFrame)
    */
    
    let imageMaterial = SCNMaterial()
    imageMaterial.isDoubleSided = false
    imageMaterial.diffuse.contents = texture!
    nodeGeometry.materials = [imageMaterial]
    
    return nodeGeometry
}

我努力的地方是确定这些纹理坐标是否真正正确计算，随后，我将如何对相机帧进行采样以将相关帧图像应用为该网格的纹理。

链接的问题表明，将ARFrame's capturedImage（这是 a CVPixelBuffer）属性转换为 aMTLTexture将是实时性能的理想选择，但对我来说很明显这CVPixelBuffer是一个YCbCr图像，而我相信我需要一个RGB图像。

在我的textureConverter课堂上，我试图将CVPixelBuffera转换为 a MTLTexture，但不确定如何返回 a RGB MTLTexture；

func makeTextureForMeshModel(frame: ARFrame) -> MTLTexture? {
    if CVPixelBufferGetPlaneCount(frame.capturedImage) < 2 {
        return nil
    }
    let cameraImageTextureY = createTexture(fromPixelBuffer: frame.capturedImage, pixelFormat: .r8Unorm, planeIndex: 0)
    let cameraImageTextureCbCr = createTexture(fromPixelBuffer: frame.capturedImage, pixelFormat: .rg8Unorm, planeIndex: 1)
    
    /* How do I blend the Y and CbCr textures, or return a RGB texture, to return a single MTLTexture?
    return ...
}

func createTexture(fromPixelBuffer pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, planeIndex: Int) -> CVMetalTexture? {
    let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex)
    let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex)
    
    var texture: CVMetalTexture? = nil
    let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, pixelFormat,
                                                           width, height, planeIndex, &texture)
    
    if status != kCVReturnSuccess {
        texture = nil
    }
    
    return texture
}

最后，我不完全确定我是否真的需要RGB纹理与YCbCr纹理，但我仍然不确定我将如何返回正确的纹理图像（我尝试只返回CVPixelBuffer而不担心YCbCr颜色空间，通过手动设置纹理格式，会产生非常奇怪的图像）。

Answer 1

您可以在这里查看我的存储库：MetalWorldTextureScan

该项目演示了如何：

• 扫描时使用 Metal 渲染网格
• 将扫描件裁剪到边界框
• 保存相机帧以进行纹理处理
• 从扫描中创建 SCNGeometry 并对其进行纹理化

计算纹理坐标：

Frame：保存用于纹理的帧

vert：要投影到框架中的顶点

aTrans：顶点所属的网格“块”的变换

func getTextureCoord(frame: ARFrame, vert: SIMD3<Float>, aTrans: simd_float4x4) -> vector_float2 {
    
    // convert vertex to world coordinates
    let cam = frame.camera
    let size = cam.imageResolution
    let vertex4 = vector_float4(vert.x, vert.y, vert.z, 1)
    let world_vertex4 = simd_mul(aTrans, vertex4)
    let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
    
    // project the point into the camera image to get u,v
    let pt = cam.projectPoint(world_vector3,
        orientation: .portrait,
        viewportSize: CGSize(
            width: CGFloat(size.height),
            height: CGFloat(size.width)))
    let v = 1.0 - Float(pt.x) / Float(size.height)
    let u = Float(pt.y) / Float(size.width)
    
    let tCoord = vector_float2(u, v)
    
    return tCoord
}

保存纹理框架：

名为“TextureFrame”的结构用于保存位置、ARFrame 以及其他可能有用的信息。

struct TextureFrame {
    var key: String       // date/time/anything
    var dist: CGFloat     // dist from bBox
    var frame: ARFrame    // saved frame
    var pos: SCNVector3   // location in reference to bBox
}

使用方法：

func saveTextureFrame() {
    guard let frame = session.currentFrame else {
        print("can't get current frame")
        return
    }
    
    let camTrans = frame.camera.transform
    let camPos = SCNVector3(camTrans.columns.3.x, camTrans.columns.3.y, camTrans.columns.3.z)
    let cam2BoxLocal = SCNVector3(camPos.x - bBoxOrigin.x, camPos.y - bBoxOrigin.y, camPos.z - bBoxOrigin.z)
    let dist = dist3D(a: camPos, b: bBoxOrigin)
    
    let dateFormatter = DateFormatter()
    dateFormatter.dateFormat = "yyyy:MM:dd:HH:mm:ss:SS"
    dateFormatter.timeZone = TimeZone(abbreviation: "CDT")
    let date = Date()
    let dString = dateFormatter.string(from: date)
    
    let textFrame = TextureFrame(key: dString, dist: dist, frame: frame, pos: cam2BoxLocal)
    textureCloud.append(textFrame)
    delegate.didSaveFrame(renderer: self)
}

制作纹理网格：

这发生在makeTexturedMesh()函数中。我们迭代所有网格块，并迭代每个块的每个面（三角形），其中计算纹理坐标，并为三角形创建单个 SCNGeometry 并将其添加到场景中。还定义了recombineGeometries()函数，用于重新组合三角形。

func makeTexturedMesh() {
    
    let worldMeshes = renderer.worldMeshes
    let textureCloud = renderer.textureCloud
    
    print("texture images: \(textureImgs.count)")
    
    // each 'mesh' is a chunk of the whole scan
    for mesh in worldMeshes {
        
        let aTrans = SCNMatrix4(mesh.transform)
        
        let vertices: ARGeometrySource = mesh.vertices
        let normals: ARGeometrySource = mesh.normals
        let faces: ARGeometryElement = mesh.submesh
        
        var texture: UIImage!
        
        // a face is just a list of three indices, each representing a vertex
        for f in 0..<faces.count {
            
            // check to see if each vertex of the face is inside of our box
            var c = 0
            let face = face(at: f, faces: faces)
            for fv in face {
                // this is set by the renderer
                if mesh.inBox[fv] == 1 {
                    c += 1
                }
            }
            
            guard c == 3 else {continue}
            
            // all verts of the face are in the box, so the triangle is visible
            var fVerts: [SCNVector3] = []
            var fNorms: [SCNVector3] = []
            var tCoords: [vector_float2] = []
            
            // convert each vertex and normal to world coordinates
            // get the texture coordinates
            for fv in face {
                
                let vert = vertex(at: UInt32(fv), vertices: vertices)
                let vTrans = SCNMatrix4MakeTranslation(vert[0], vert[1], vert[2])
                let wTrans = SCNMatrix4Mult(vTrans, aTrans)
                let wPos = SCNVector3(wTrans.m41, wTrans.m42, wTrans.m43)
                fVerts.append(wPos)
                
                let norm = normal(at: UInt32(fv), normals: normals)
                let nTrans = SCNMatrix4MakeTranslation(norm[0], norm[1], norm[2])
                let wNTrans = SCNMatrix4Mult(nTrans, aTrans)
                let wNPos = SCNVector3(wNTrans.m41, wTrans.m42, wNTrans.m43)
                fNorms.append(wNPos)
                
                
                // here's where you would find the frame that best fits
                // for simplicity, just use the last frame here
                let tFrame = textureCloud.last!.frame
                let tCoord = getTextureCoord(frame: tFrame, vert: vert, aTrans: mesh.transform)
                tCoords.append(tCoord)
                texture = textureImgs[textureCloud.count - 1]
                
                // visualize the normals if you want
                if mesh.inBox[fv] == 1 {
                    //let normVis = lineBetweenNodes(positionA: wPos, positionB: wNPos, inScene: arView.scene)
                    //arView.scene.rootNode.addChildNode(normVis)
                }
            }
            allVerts.append(fVerts)
            allNorms.append(fNorms)
            allTCrds.append(tCoords)
            
            // make a single triangle mesh out each face
            let vertsSource = SCNGeometrySource(vertices: fVerts)
            let normsSource = SCNGeometrySource(normals: fNorms)
            let facesSource = SCNGeometryElement(indices: [UInt32(0), UInt32(1), UInt32(2)], primitiveType: .triangles)
            let textrSource = SCNGeometrySource(textureCoordinates: tCoords)
            let geom = SCNGeometry(sources: [vertsSource, normsSource, textrSource], elements: [facesSource])
            
            // texture it with a saved camera frame
            let mat = SCNMaterial()
            mat.diffuse.contents = texture
            mat.isDoubleSided = false
            geom.materials = [mat]
            let meshNode = SCNNode(geometry: geom)
            
            DispatchQueue.main.async {
                self.scanNode.addChildNode(meshNode)
            }
        }
    }
}

该项目还包括从文档目录保存和加载网格的方法。

这绝不是对 3D 扫描进行网格划分和纹理化的最佳方法，但它很好地演示了如何开始使用内置 iOS 框架。