Merging visionos branch into main (#87)

maksutovic · wtholliday · aure · web-flow · commit 3559d8b7aa28 · 2024-04-20T16:53:56.000-07:00
* Move code to extension

* Don't have MTKView on VisionOS

* Add encode function and draw(in:)

* Resize waveformTexture

* Use drawable size for width of waveformTexture

* Move code out

* FloatPlot shouldn't be an MTKView

* Make private

* Add MetalView

* Add createDisplayLink

* Logging

* Add logging

* visionOS support

* uncomment display lnk

* Update FloatPlot.swift

* Fix warning

* Fix warnings

* Fix macOS build

* visionOS support

Co-Authored-By: Aurelius Prochazka &lt;aure@aure.com&gt;

* FloatPloat now defaults to a clear background

---------

Co-authored-by: Taylor Holliday &lt;wtholliday@gmail.com&gt;
Co-authored-by: Aurelius Prochazka &lt;aure@aure.com&gt;
diff --git a/Sources/AudioKitUI/Controls/ADSRView.swift b/Sources/AudioKitUI/Controls/ADSRView.swift
@@ -502,7 +502,7 @@ extension CGPoint {
 #else
 
 import AVFoundation
-import Cocoa
+import AppKit
 
 /// Call back for values for attack, decay, sustain, and release parameters
 public typealias ADSRCallback = (AUValue, AUValue, AUValue, AUValue) -> Void
diff --git a/Sources/AudioKitUI/Visualizations/FloatPlot.swift b/Sources/AudioKitUI/Visualizations/FloatPlot.swift
@@ -5,7 +5,7 @@ import Metal
 import MetalKit
 
 // This must be in sync with the definition in shaders.metal
-public struct FragmentConstants {
+struct FragmentConstants {
     public var foregroundColor: SIMD4<Float>
     public var backgroundColor: SIMD4<Float>
     public var isFFT: Bool
@@ -17,13 +17,15 @@ public struct FragmentConstants {
     public var padding: Int = 0
 }
 
-public class FloatPlot: MTKView, MTKViewDelegate {
-    let waveformTexture: MTLTexture!
+class FloatPlot: NSObject {
+    var waveformTexture: MTLTexture?
     let commandQueue: MTLCommandQueue!
     let pipelineState: MTLRenderPipelineState!
-    let bufferSampleCount: Int
+    var bufferSampleCount: Int
     var dataCallback: () -> [Float]
     var constants: FragmentConstants
+    let layerRenderPassDescriptor: MTLRenderPassDescriptor
+    let device = MTLCreateSystemDefaultDevice()
 
     public init(frame frameRect: CGRect,
                 constants: FragmentConstants,
@@ -32,15 +34,6 @@ public class FloatPlot: MTKView, MTKViewDelegate {
         self.constants = constants
         bufferSampleCount = Int(frameRect.width)
 
-        let desc = MTLTextureDescriptor()
-        desc.textureType = .type1D
-        desc.width = Int(frameRect.width)
-        desc.pixelFormat = .r32Float
-        assert(desc.height == 1)
-        assert(desc.depth == 1)
-
-        let device = MTLCreateSystemDefaultDevice()
-        waveformTexture = device?.makeTexture(descriptor: desc)
         commandQueue = device!.makeCommandQueue()
 
         let library = try! device?.makeDefaultLibrary(bundle: Bundle.module)
@@ -51,7 +44,6 @@ public class FloatPlot: MTKView, MTKViewDelegate {
         let pipelineStateDescriptor = MTLRenderPipelineDescriptor()
         pipelineStateDescriptor.vertexFunction = vertexProgram
         pipelineStateDescriptor.fragmentFunction = fragmentProgram
-        pipelineStateDescriptor.sampleCount = 1
 
         let colorAttachment = pipelineStateDescriptor.colorAttachments[0]!
         colorAttachment.pixelFormat = .bgra8Unorm
@@ -63,23 +55,45 @@ public class FloatPlot: MTKView, MTKViewDelegate {
 
         pipelineState = try! device!.makeRenderPipelineState(descriptor: pipelineStateDescriptor)
 
-        super.init(frame: frameRect, device: device)
-
-        clearColor = .init(red: 0.0, green: 0.0, blue: 0.0, alpha: 0)
-
-        delegate = self
+        layerRenderPassDescriptor = MTLRenderPassDescriptor()
+        layerRenderPassDescriptor.colorAttachments[0].loadAction = .clear
+        layerRenderPassDescriptor.colorAttachments[0].storeAction = .store
+        layerRenderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0);
     }
 
     @available(*, unavailable)
     required init(coder _: NSCoder) {
         fatalError("init(coder:) has not been implemented")
     }
 
+    func resize(width: Int) {
+
+        if width == 0 {
+            return
+        }
+
+        let desc = MTLTextureDescriptor()
+        desc.textureType = .type1D
+        desc.width = width
+        desc.pixelFormat = .r32Float
+        assert(desc.height == 1)
+        assert(desc.depth == 1)
+
+        waveformTexture = device?.makeTexture(descriptor: desc)
+        bufferSampleCount = width
+
+    }
+
     func updateWaveform(samples: [Float]) {
         if samples.count == 0 {
             return
         }
 
+        guard let waveformTexture else {
+            print("⚠️ updateWaveform: waveformTexture is nil")
+            return
+        }
+
         var resampled = [Float](repeating: 0, count: bufferSampleCount)
 
         for i in 0 ..< bufferSampleCount {
@@ -97,24 +111,60 @@ public class FloatPlot: MTKView, MTKViewDelegate {
         }
     }
 
-    public func mtkView(_: MTKView, drawableSizeWillChange _: CGSize) {
-        // We may want to resize the texture.
+    func encode(to commandBuffer: MTLCommandBuffer, pass: MTLRenderPassDescriptor) {
+        guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) else { return }
+
+        encoder.setRenderPipelineState(pipelineState)
+        encoder.setFragmentTexture(waveformTexture, index: 0)
+        assert(MemoryLayout<FragmentConstants>.size == 48)
+        encoder.setFragmentBytes(&constants, length: MemoryLayout<FragmentConstants>.size, index: 0)
+        encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
+        encoder.endEncoding()
     }
 
-    public func draw(in view: MTKView) {
+    func draw(to layer: CAMetalLayer) {
+
         updateWaveform(samples: dataCallback())
+        
+        let size = layer.drawableSize
+        let w = Float(size.width)
+        let h = Float(size.height)
+        // let scale = Float(view.contentScaleFactor)
 
-        if let commandBuffer = commandQueue.makeCommandBuffer() {
-            if let renderPassDescriptor = currentRenderPassDescriptor {
-                guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }
+        if w == 0 || h == 0 {
+            return
+        }
+
+        guard let commandBuffer = commandQueue.makeCommandBuffer() else {
+            return
+        }
+
+        if let currentDrawable = layer.nextDrawable() {
+
+            layerRenderPassDescriptor.colorAttachments[0].texture = currentDrawable.texture
 
-                encoder.setRenderPipelineState(pipelineState)
-                encoder.setFragmentTexture(waveformTexture, index: 0)
-                assert(MemoryLayout<FragmentConstants>.size == 48)
-                encoder.setFragmentBytes(&constants, length: MemoryLayout<FragmentConstants>.size, index: 0)
-                encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
-                encoder.endEncoding()
+            encode(to: commandBuffer, pass: layerRenderPassDescriptor)
 
+            commandBuffer.present(currentDrawable)
+        } else {
+            print("⚠️ couldn't get drawable")
+        }
+        commandBuffer.commit()
+    }
+}
+
+#if !os(visionOS)
+extension FloatPlot: MTKViewDelegate {
+    public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
+        resize(width: Int(size.width))
+    }
+
+    public func draw(in view: MTKView) {
+        updateWaveform(samples: dataCallback())
+
+        if let commandBuffer = commandQueue.makeCommandBuffer() {
+            if let renderPassDescriptor = view.currentRenderPassDescriptor {
+                encode(to: commandBuffer, pass: renderPassDescriptor)
                 if let drawable = view.currentDrawable {
                     commandBuffer.present(drawable)
                 }
@@ -125,3 +175,38 @@ public class FloatPlot: MTKView, MTKViewDelegate {
         }
     }
 }
+#endif
+
+#if !os(visionOS)
+public class FloatPlotCoordinator {
+    var renderer: FloatPlot
+
+    init(renderer: FloatPlot) {
+        self.renderer = renderer
+    }
+
+    var view: MTKView {
+        let view = MTKView(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024), device: renderer.device)
+        view.clearColor = .init(red: 0.0, green: 0.0, blue: 0.0, alpha: 0)
+        view.delegate = renderer
+        return view
+    }
+}
+#else
+public class FloatPlotCoordinator {
+    var renderer: FloatPlot
+
+    init(renderer: FloatPlot) {
+        self.renderer = renderer
+    }
+
+    var view: MetalView {
+        let view = MetalView(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024))
+        view.renderer = renderer
+        view.metalLayer.pixelFormat = .bgra8Unorm
+        view.metalLayer.isOpaque = false
+        view.createDisplayLink()
+        return view
+    }
+}
+#endif
diff --git a/Sources/AudioKitUI/Visualizations/MetalView.swift b/Sources/AudioKitUI/Visualizations/MetalView.swift
@@ -0,0 +1,90 @@
+// Copyright AudioKit. All Rights Reserved. Revision History at http://github.com/AudioKit/Waveform/
+
+#if os(iOS) || os(visionOS)
+import UIKit
+
+class MetalView: UIView {
+
+    var renderer: FloatPlot?
+    var displayLink: CADisplayLink?
+
+    @objc static override var layerClass: AnyClass {
+        CAMetalLayer.self
+    }
+
+    var metalLayer: CAMetalLayer {
+        layer as! CAMetalLayer
+    }
+
+    func createDisplayLink() {
+        displayLink = CADisplayLink(target: self,
+                                    selector: #selector(render))
+
+        displayLink?.add(to: .current,
+                         forMode: .default)
+    }
+
+    override func draw(_ rect: CGRect) {
+        render()
+    }
+
+    override func draw(_ layer: CALayer, in ctx: CGContext) {
+        render()
+    }
+
+    override func display(_ layer: CALayer) {
+        render()
+    }
+
+    @objc func render() {
+        guard let renderer else {
+            print("⚠️ no renderer")
+            return
+        }
+        renderer.draw(to: metalLayer)
+    }
+
+    func resizeDrawable() {
+
+        var newSize = bounds.size
+        newSize.width *= contentScaleFactor
+        newSize.height *= contentScaleFactor
+
+        if newSize.width <= 0 || newSize.height <= 0 {
+            return
+        }
+
+        if newSize.width == metalLayer.drawableSize.width &&
+            newSize.height == metalLayer.drawableSize.height {
+            return
+        }
+
+        metalLayer.drawableSize = newSize
+        renderer?.resize(width: Int(newSize.width))
+
+        setNeedsDisplay()
+    }
+
+    @objc override var frame: CGRect {
+        get { super.frame }
+        set {
+            super.frame = newValue
+            resizeDrawable()
+        }
+    }
+
+    @objc override func layoutSubviews() {
+        super.layoutSubviews()
+        resizeDrawable()
+    }
+
+    @objc override var bounds: CGRect {
+        get { super.bounds }
+        set {
+            super.bounds = newValue
+            resizeDrawable()
+        }
+    }
+
+}
+#endif
diff --git a/Sources/AudioKitUI/Visualizations/NodeFFTView.swift b/Sources/AudioKitUI/Visualizations/NodeFFTView.swift
@@ -4,6 +4,7 @@ import Accelerate
 import AudioKit
 import AVFoundation
 import SwiftUI
+import MetalKit
 
 public struct NodeFFTView: ViewRepresentable {
     var nodeTap: FFTTap
@@ -13,7 +14,7 @@ public struct NodeFFTView: ViewRepresentable {
         nodeTap = FFTTap(node, bufferSize: UInt32(bufferSampleCount), callbackQueue: .main) { _ in }
     }
 
-    internal var plot: FloatPlot {
+    public func makeCoordinator() -> FloatPlotCoordinator {
         nodeTap.start()
 
         let constants = FragmentConstants(foregroundColor: Color.yellow.simd,
@@ -26,14 +27,14 @@ public struct NodeFFTView: ViewRepresentable {
             nodeTap.fftData
         }
 
-        return plot
+        return .init(renderer: plot)
     }
 
     #if os(macOS)
-    public func makeNSView(context: Context) -> FloatPlot { return plot }
-    public func updateNSView(_ nsView: FloatPlot, context: Context) {}
+    public func makeNSView(context: Context) -> NSView { return context.coordinator.view }
+    public func updateNSView(_ nsView: NSView, context: Context) {}
     #else
-    public func makeUIView(context: Context) -> FloatPlot { return plot }
-    public func updateUIView(_ uiView: FloatPlot, context: Context) {}
+    public func makeUIView(context: Context) -> UIView { return context.coordinator.view }
+    public func updateUIView(_ uiView: UIView, context: Context) {}
     #endif
 }
diff --git a/Sources/AudioKitUI/Visualizations/NodeOutputView.swift b/Sources/AudioKitUI/Visualizations/NodeOutputView.swift
@@ -4,6 +4,7 @@ import Accelerate
 import AudioKit
 import AVFoundation
 import SwiftUI
+import MetalKit
 
 public struct NodeOutputView: ViewRepresentable {
     private var nodeTap: RawDataTap
@@ -18,19 +19,21 @@ public struct NodeOutputView: ViewRepresentable {
         nodeTap = RawDataTap(node, bufferSize: UInt32(bufferSize), callbackQueue: .main)
     }
 
-    var plot: FloatPlot {
+    public func makeCoordinator() -> FloatPlotCoordinator {
         nodeTap.start()
 
-        return FloatPlot(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024), constants: constants) {
-            return nodeTap.data
+        let plot = FloatPlot(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024), constants: constants) {
+            nodeTap.data
         }
+
+        return .init(renderer: plot)
     }
 
     #if os(macOS)
-    public func makeNSView(context: Context) -> FloatPlot { return plot }
-    public func updateNSView(_ nsView: FloatPlot, context: Context) {}
+    public func makeNSView(context: Context) -> NSView { return context.coordinator.view }
+    public func updateNSView(_ nsView: NSView, context: Context) {}
     #else
-    public func makeUIView(context: Context) -> FloatPlot { return plot }
-    public func updateUIView(_ uiView: FloatPlot, context: Context) {}
+    public func makeUIView(context: Context) -> UIView { return context.coordinator.view }
+    public func updateUIView(_ uiView: UIView, context: Context) {}
     #endif
 }
diff --git a/Sources/AudioKitUI/Visualizations/NodeRollingView.swift b/Sources/AudioKitUI/Visualizations/NodeRollingView.swift
diff --git a/Sources/AudioKitUI/Visualizations/TableView.swift b/Sources/AudioKitUI/Visualizations/TableView.swift
