I am experimenting with an AR app. I am trying to do the following:
The iOS device displays the real scene by an ARView, and the ARView creates a mesh.
Whenever the mesh is updated, I want to find the vertex closest to the camera, and attach a virtual object to it (after deleting a possibly previously attached object).
I am not sure if my current code is correct, but it seems that it does what I have described above. If I move the device, the displayed mesh is updated. When a new closest vertex is found, it toggles a state var refreshToggle in the MainView, and I expected that the updated mesh is displayed together with the virtual object. But the virtual object is not shown, and I don't understand why.
Here is my code. I am sorry that it is so long, but I am not sure what to omit. Any help is welcome!
struct MainView : View {
@State private var refreshToggle = false // Toggled, when a new closest anchor is found
var body: some View {
ARViewContainer(refreshToggle: $refreshToggle).edgesIgnoringSafeArea(.all)
}
}
struct ARViewContainer: UIViewRepresentable {
@Binding var refreshToggle: Bool
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
arView.environment.sceneUnderstanding.options = []
arView.environment.sceneUnderstanding.options.insert(.occlusion) // Turn on occlusion from the scene reconstruction's mesh.
arView.environment.sceneUnderstanding.options.insert(.physics) // Turn on physics for the scene reconstruction's mesh.
arView.debugOptions.insert(.showSceneUnderstanding) // Display a debug visualization of the mesh.
arView.renderOptions = [.disablePersonOcclusion, .disableDepthOfField, .disableMotionBlur] // Disable not required render options
arView.session.delegate = context.coordinator
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {}
func makeCoordinator() -> Coordinator {
Coordinator($refreshToggle)
}
class Coordinator: NSObject, ARSessionDelegate {
@Binding var refreshToggle: Bool
var model: ModelEntity
init(_ refreshToggle: Binding<Bool>) {
self._refreshToggle = refreshToggle
// Create a cube model
let mesh = MeshResource.generateBox(size: 0.1, cornerRadius: 0.005)
let material = SimpleMaterial(color: .gray, roughness: 0.15, isMetallic: true)
model = ModelEntity(mesh: mesh, materials: [material])
}
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
var closestAnchor: ARAnchor? = nil
for anchor in anchors {
if let meshAnchor = anchor as? ARMeshAnchor {
let meshGeometry = meshAnchor.geometry
let vertices = meshGeometry.vertices
// Search for the vertex closest to the camera and place there a virtual marker object
let nrVertices = vertices.count
var closestVertex = SIMD3<Float>(x: 0, y: .infinity, z: 0)
for i in 0 ..< nrVertices {
let nextVertex = meshGeometry.vertex(at: UInt32(i))
if nextVertex.y < closestVertex.y {
closestVertex = nextVertex
if closestAnchor?.identifier != meshAnchor.identifier {
// A new closest anchor has been found. Remove a virtual marker object
if let closestAnchor = closestAnchor {
let anchor = AnchorEntity(anchor: closestAnchor)
anchor.children.remove(model)
}
}
closestAnchor = meshAnchor
}
}
// If a closest vertex was found, attach a virtual object to it
if let closestAnchor = closestAnchor {
let anchor = AnchorEntity(anchor: closestAnchor)
anchor.children.append(model)
refreshToggle = !refreshToggle // Let ARViewContainer redisplay the real scene with the mesh and a virtual object attached to the closest anchor
}
} // if an ARMeshAnchor was found
} // for all anchors
} // session didUpdate anchors
} // coordinator
}
extension ARMeshGeometry { // See https://developer.apple.com/documentation/arkit/armeshgeometry/3516924-vertices
func vertex(at index: UInt32) -> SIMD3<Float> {
assert(vertices.format == MTLVertexFormat.float3, "Expected three floats (twelve bytes) per vertex.")
let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + (vertices.stride * Int(index)))
let vertex = vertexPointer.assumingMemoryBound(to: SIMD3<Float>.self).pointee
return vertex
}
}
Problem solved, although I don't understand it (no experience in Computer Graphics).
After the closest mesh anchor has been found, one has to create an AnchorEntity.
In this entity, one has to set the anchoring property with an AnchoringComponent initialized with an appropriate AnchoringComponent.Target. Only then is the virtual object rendered in the scene.
The following code works for me, and is based on some valuable info, the answer of KFDoom (+1), a blog of Ethan Saadia, and a tutorial of Ralf Ebert.
Here is the updated code in case somebody wants to play with it.
The transform in .world(transform: transform) has been taken from a different version and turned out to be useful.
import ARKit
import RealityKit
import SwiftUI
struct MainView: View {
var body: some View {
ARViewContainer()
.edgesIgnoringSafeArea(.all)
}
}
struct ARViewContainer: UIViewRepresentable {
func makeUIView(context: Context) -> ARView {
let arView = ARView()
// Configure the ARView to generate a mesh
arView.environment.sceneUnderstanding.options = []
// Turn on occlusion from the scene reconstruction's mesh.
arView.environment.sceneUnderstanding.options.insert(.occlusion)
// Turn on physics for the scene reconstruction's mesh.
arView.environment.sceneUnderstanding.options.insert(.physics)
// Display a debug visualization of the mesh.
arView.debugOptions.insert(.showSceneUnderstanding)
// For performance, disable render options that are not required for this app.
arView.renderOptions = [.disablePersonOcclusion, .disableDepthOfField, .disableMotionBlur]
arView.session.delegate = context.coordinator
// Handle ARSession events via delegate
context.coordinator.arView = arView
arView.session.delegate = context.coordinator
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {}
func makeCoordinator() -> Coordinator {
Coordinator()
}
class Coordinator: NSObject, ARSessionDelegate {
var model: ModelEntity
weak var arView: ARView?
override init() {
// Create a cube model
let boxMesh = MeshResource.generateBox(size: 0.1, cornerRadius: 0.005)
let material = SimpleMaterial(color: .gray, roughness: 0.15, isMetallic: true)
model = ModelEntity(mesh: boxMesh, materials: [material])
super.init()
}
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
var closestAnchor: ARAnchor? = nil
guard let arView = arView else { return }
// Create a single AnchorEntity instance
var anchorEntity: AnchorEntity?
for anchor in anchors {
if let meshAnchor = anchor as? ARMeshAnchor {
let meshGeometry = meshAnchor.geometry
let vertices = meshGeometry.vertices
// For debugging, we search for the vertex closest to the camera and place there a virtual marker object
let nrVertices = vertices.count
var closestVertex = SIMD3<Float>(x: 0, y: .infinity, z: 0)
for i in 0 ..< nrVertices {
let nextVertex = meshGeometry.vertex(at: UInt32(i))
// The frontmost vertex has the largest z value, see https://developer.apple.com/documentation/scenekit/organizing_a_scene_with_nodes
if nextVertex.z > closestVertex.z {
closestVertex = nextVertex
if closestAnchor?.identifier != meshAnchor.identifier {
// A new closest anchor has been found. Remove the virtual marker object if it exists.
// If an anchorEntity already exists, remove it from the ARView's scene
if let existingAnchor = anchorEntity {
existingAnchor.removeFromParent()
}
}
closestAnchor = meshAnchor
}
}
} // if an ARMeshAnchor found
} // for all anchors
// If a closest vertex was found, attach a virtual object to it
if let closestAnchor = closestAnchor {
// Create a new AnchorEntity and attach the model to it
anchorEntity = AnchorEntity(anchor: closestAnchor)
let transform = simd_float4x4([[0.96475136, 0.0, 0.26316252, 0.0], [0.0, 1.0, 0.0, 0.0], [-0.26316252, 0.0, 0.9647514, 0.0], [0.16189954, -0.25364277, -0.22894737, 1.0]])
let anchoring = AnchoringComponent(.world(transform: transform))
anchorEntity!.anchoring = anchoring
anchorEntity!.addChild(model)
arView.scene.anchors.append(anchorEntity!)
}
} // session didUpdate anchors
} // coordinator
}
extension ARMeshGeometry { // See https://developer.apple.com/documentation/arkit/armeshgeometry/3516924-vertices
func vertex(at index: UInt32) -> SIMD3<Float> {
assert(vertices.format == MTLVertexFormat.float3, "Expected three floats (twelve bytes) per vertex.")
let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + (vertices.stride * Int(index)))
let vertex = vertexPointer.assumingMemoryBound(to: SIMD3<Float>.self).pointee
return vertex
}
}
#Preview {
MainView()
}