Extract io_surface crate and invoke custom callback on frame sample

Co-Authored-By: Nathan Sobo <nathan@zed.dev>

Cargo.lock

@@ -762,6 +762,7 @@ dependencies = [
  "core-graphics",
  "foreign-types",
  "gpui",
+ "io_surface",
  "log",
  "objc",
  "simplelog",
@@ -2228,6 +2229,7 @@ dependencies = [
  "futures",
  "gpui_macros",
  "image",
+ "io_surface",
  "lazy_static",
  "log",
  "metal",
@@ -2596,6 +2598,15 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "io_surface"
+version = "0.1.0"
+dependencies = [
+ "block",
+ "core-foundation",
+ "objc",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"

crates/capture/Cargo.toml

@@ -10,6 +10,7 @@ identifier = "dev.zed.Capture"
 
 [dependencies]
 gpui = { path = "../gpui" }
+io_surface = { path = "../io_surface" }
 
 block = "0.1"
 cocoa = "0.24"

crates/capture/build.rs

@@ -24,8 +24,9 @@ fn main() {
         .allowlist_function("CMTimeMake")
         .allowlist_type("SCStreamOutputType")
         .allowlist_type("SCFrameStatus")
+        .allowlist_type("dispatch_queue_t")
         .allowlist_var("SCStreamFrameInfo.*")
-        .allowlist_function("dispatch_queue_create")
+        .allowlist_var("_dispatch_main_q")
         .parse_callbacks(Box::new(bindgen::CargoCallbacks))
         .layout_tests(false)
         .generate()

crates/capture/src/bindings.rs

@@ -1,7 +1,12 @@
 #![allow(non_upper_case_globals)]
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
+#![allow(unused)]
 
 use objc::*;
 
 include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+
+pub fn dispatch_get_main_queue() -> dispatch_queue_t {
+    unsafe { std::mem::transmute(&_dispatch_main_q) }
+}

crates/capture/src/main.rs

@@ -1,6 +1,6 @@
 mod bindings;
 
-use crate::bindings::{dispatch_queue_create, NSObject, SCStreamOutputType};
+use crate::bindings::{dispatch_get_main_queue, SCStreamOutputType};
 use block::ConcreteBlock;
 use cocoa::{
     base::{id, nil, YES},
@@ -8,6 +8,7 @@ use cocoa::{
 };
 use core_foundation::{base::TCFType, number::CFNumberRef, string::CFStringRef};
 use core_media::{CMSampleBuffer, CMSampleBufferRef};
+use gpui::elements::Canvas;
 use gpui::{actions, elements::*, keymap::Binding, Menu, MenuItem};
 use log::LevelFilter;
 use objc::{
@@ -18,7 +19,7 @@ use objc::{
     sel, sel_impl,
 };
 use simplelog::SimpleLogger;
-use std::{ptr, slice, str};
+use std::{ffi::c_void, slice, str};
 
 #[allow(non_upper_case_globals)]
 const NSUTF8StringEncoding: NSUInteger = 4;
@@ -55,11 +56,17 @@ fn main() {
                 let display_height: usize = msg_send![display, height];
 
                 let mut decl = ClassDecl::new("CaptureOutput", class!(NSObject)).unwrap();
+                decl.add_ivar::<*mut c_void>("callback");
                 decl.add_method(sel!(stream:didOutputSampleBuffer:ofType:), sample_output as extern "C" fn(&Object, Sel, id, id, SCStreamOutputType));
                 let capture_output_class = decl.register();
 
                 let output: id = msg_send![capture_output_class, alloc];
                 let output: id = msg_send![output, init];
+                let callback = Box::new(|buffer: CMSampleBufferRef| {
+                    dbg!("HEY!");
+                }) as Box<dyn FnMut(CMSampleBufferRef)>;
+                let callback = Box::into_raw(Box::new(callback));
+                (*output).set_ivar("callback", callback as *mut c_void);
 
                 let filter: id = msg_send![class!(SCContentFilter), alloc];
                 let filter: id = msg_send![filter, initWithDisplay: display includingApplications: applications exceptingWindows: nil];
@@ -75,7 +82,7 @@ fn main() {
                 let stream: id = msg_send![class!(SCStream), alloc];
                 let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output];
                 let error: id = nil;
-                let queue = dispatch_queue_create(ptr::null(), NSObject(ptr::null_mut()));
+                let queue = dispatch_get_main_queue();
 
                 let _: () = msg_send![stream,
                     addStreamOutput: output type: bindings::SCStreamOutputType_SCStreamOutputTypeScreen
@@ -107,19 +114,29 @@ fn main() {
     });
 }
 
-struct ScreenCaptureView;
+struct ScreenCaptureView {
+    surface: io_surface::IOSurface,
+}
 
 impl gpui::Entity for ScreenCaptureView {
     type Event = ();
 }
 
+// impl ScreenCaptureView {
+//     pub fn new() -> Self {}
+// }
+
 impl gpui::View for ScreenCaptureView {
     fn ui_name() -> &'static str {
         "View"
     }
 
     fn render(&mut self, _: &mut gpui::RenderContext<Self>) -> gpui::ElementBox {
-        Empty::new().boxed()
+        Canvas::new(|bounds, _, cx| {
+
+            // cx.scene.push_surface(surface)
+        })
+        .boxed()
     }
 }
 
@@ -136,13 +153,18 @@ pub unsafe fn string_from_objc(string: id) -> String {
 }
 
 extern "C" fn sample_output(
-    _: &Object,
+    this: &Object,
     _: Sel,
     _stream: id,
     buffer: id,
     _kind: SCStreamOutputType,
 ) {
     let buffer = unsafe { CMSampleBuffer::wrap_under_get_rule(buffer as CMSampleBufferRef) };
+    let callback = unsafe { *this.get_ivar::<*mut c_void>("callback") };
+    let callback = callback as *mut Box<dyn FnMut(CMSampleBufferRef)>;
+    let x = unsafe { callback.as_mut().unwrap() };
+    (*x)(buffer.as_concrete_TypeRef());
+
     let attachments = buffer.attachments();
     let attachments = attachments.first().expect("no attachments for sample");
 
@@ -228,11 +250,11 @@ mod core_media {
 mod core_video {
     #![allow(non_snake_case)]
 
-    use crate::io_surface::{IOSurface, IOSurfaceRef};
     use core_foundation::{
         base::{CFTypeID, TCFType},
         declare_TCFType, impl_CFTypeDescription, impl_TCFType,
     };
+    use io_surface::{IOSurface, IOSurfaceRef};
     use std::ffi::c_void;
 
     #[repr(C)]
@@ -269,26 +291,3 @@ mod core_video {
         fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize;
     }
 }
-
-mod io_surface {
-    #![allow(non_snake_case)]
-
-    use core_foundation::{
-        base::{CFTypeID, TCFType},
-        declare_TCFType, impl_CFTypeDescription, impl_TCFType,
-    };
-    use std::ffi::c_void;
-
-    #[repr(C)]
-    pub struct __IOSurface(c_void);
-    // The ref type must be a pointer to the underlying struct.
-    pub type IOSurfaceRef = *const __IOSurface;
-
-    declare_TCFType!(IOSurface, IOSurfaceRef);
-    impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID);
-    impl_CFTypeDescription!(IOSurface);
-
-    extern "C" {
-        fn IOSurfaceGetTypeID() -> CFTypeID;
-    }
-}

crates/gpui/Cargo.toml

@@ -60,6 +60,7 @@ png = "0.16"
 simplelog = "0.9"
 
 [target.'cfg(target_os = "macos")'.dependencies]
+io_surface = { path = "../io_surface" }
 anyhow = "1"
 block = "0.1"
 cocoa = "0.24"

crates/gpui/build.rs

@@ -19,7 +19,6 @@ fn generate_dispatch_bindings() {
         .header("src/platform/mac/dispatch.h")
         .allowlist_var("_dispatch_main_q")
         .allowlist_function("dispatch_async_f")
-        .allowlist_function("dispatch_queue_create")
         .parse_callbacks(Box::new(bindgen::CargoCallbacks))
         .layout_tests(false)
         .generate()

crates/gpui/src/platform/mac.rs

@@ -1,5 +1,5 @@
 mod atlas;
-pub mod dispatcher;
+mod dispatcher;
 mod event;
 mod fonts;
 mod geometry;

crates/gpui/src/platform/mac/renderer.rs

@@ -39,6 +39,7 @@ struct PathSprite {
 
 pub struct Surface {
     pub bounds: RectF,
+    pub native_surface: io_surface::IOSurface,
 }
 
 impl Renderer {

crates/io_surface/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "io_surface"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+path = "src/io_surface.rs"
+doctest = false
+
+[dependencies]
+block = "0.1"
+core-foundation = "0.9.3"
+objc = "0.2"

crates/io_surface/src/io_surface.rs

@@ -0,0 +1,21 @@
+#![allow(non_snake_case)]
+
+use core_foundation::{
+    base::{CFTypeID, TCFType},
+    declare_TCFType, impl_CFTypeDescription, impl_TCFType,
+};
+use std::ffi::c_void;
+
+#[repr(C)]
+pub struct __IOSurface(c_void);
+// The ref type must be a pointer to the underlying struct.
+pub type IOSurfaceRef = *const __IOSurface;
+
+declare_TCFType!(IOSurface, IOSurfaceRef);
+impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID);
+impl_CFTypeDescription!(IOSurface);
+
+#[link(name = "IOSurface", kind = "framework")]
+extern "C" {
+    fn IOSurfaceGetTypeID() -> CFTypeID;
+}