Initial work on vulkan video enc

.github/workflows/rust.yml

@@ -17,7 +17,7 @@ jobs:
       image: archlinux:latest
     steps:
     - name: Install system deps
-      run: pacman  --noconfirm -Syu ffmpeg rustup gcc clang pkgconf
+      run: pacman  --noconfirm -Syu ffmpeg rustup gcc clang pkgconf vulkan-headers
     - name: Install rust
       run: rustup install stable
     - uses: actions/checkout@v4

Cargo.toml

@@ -28,7 +28,7 @@ wayland-protocols = { version = "0.32", features = [
 ] }
 wayland-protocols-wlr = { version = "0.3", features = ["client"] }
 ffmpeg-next = "7.0.1"
-ffmpeg-sys-next = "7.0.0" # need direct dep on -sys to get metadata to consume in build.rs
+ffmpeg-sys-next = { version = "7.0.0", features = ["vulkan"] }
 thiserror = "1.0.38"
 human-size = "0.4.2"
 signal-hook = "0.3.15"
@@ -39,6 +39,7 @@ log = "0.4.21"
 clap_complete = "4.5.8"
 log-once = "0.4.1"
 drm = "0.14.0"
+ash = "0.38.0"
 
 
 [dev-dependencies]
@@ -48,5 +49,8 @@ nix = { version = "0.29.0", default-features = false, features = [
 ] }
 serde_json = "1.0.103"
 
+[patch.crates-io]
+ffmpeg-sys-next = { git = "https://github.com/russelltg/rust-ffmpeg-sys", branch = "vulkan"}
+
 [profile.release]
 lto = "thin"

build.rs

@@ -2,8 +2,10 @@ use std::env;
 
 fn main() {
     println!("cargo::rustc-check-cfg=cfg(ffmpeg_7_0)");
-    for (name, _value) in env::vars() {
-        if name.starts_with("DEP_FFMPEG_") {
+    println!("cargo::rustc-check-cfg=cfg(ffmpeg_7_1)");
+
+    for (name, value) in env::vars() {
+        if name.starts_with("DEP_FFMPEG_") && !value.is_empty() {
             println!(
                 r#"cargo::rustc-cfg={}"#,
                 name["DEP_FFMPEG_".len()..name.len()].to_lowercase()

src/avhw.rs

@@ -1,5 +1,6 @@
 use std::{ffi::CString, path::Path, ptr::null_mut};
 
+use ash::vk;
 use ffmpeg::{
     dict,
     ffi::{
@@ -9,9 +10,13 @@ use ffmpeg::{
     format::Pixel,
     frame,
 };
+use ffmpeg_sys_next::AVVulkanFramesContext;
+
+use crate::DrmModifier;
 
 pub struct AvHwDevCtx {
     ptr: *mut ffmpeg::sys::AVBufferRef,
+    fmt: Pixel,
 }
 
 impl AvHwDevCtx {
@@ -35,7 +40,34 @@ impl AvHwDevCtx {
             if sts != 0 {
                 Err(ffmpeg::Error::from(sts))
             } else {
-                Ok(Self { ptr: hw_device_ctx })
+                Ok(Self {
+                    ptr: hw_device_ctx,
+                    fmt: Pixel::VAAPI,
+                })
+            }
+        }
+    }
+
+    pub fn new_vulkan(dri_device: &str) -> Result<Self, ffmpeg::Error> {
+        unsafe {
+            let mut hw_device_ctx = null_mut();
+
+            let dev_cstr = CString::new(dri_device).unwrap();
+            let sts = av_hwdevice_ctx_create(
+                &mut hw_device_ctx,
+                ffmpeg_next::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN,
+                dev_cstr.as_ptr(),
+                null_mut(),
+                0,
+            );
+
+            if sts != 0 {
+                Err(ffmpeg::Error::from(sts))
+            } else {
+                Ok(Self {
+                    ptr: hw_device_ctx,
+                    fmt: Pixel::VULKAN,
+                })
             }
         }
     }
@@ -45,17 +77,31 @@ impl AvHwDevCtx {
         pixfmt: Pixel,
         width: i32,
         height: i32,
+        modifiers: &[DrmModifier],
     ) -> Result<AvHwFrameCtx, ffmpeg::Error> {
         unsafe {
             let mut hwframe = av_hwframe_ctx_alloc(self.ptr as *mut _);
-            let hwframe_casted = (*hwframe).data as *mut AVHWFramesContext;
+            let hwframe_casted = &mut *((*hwframe).data as *mut AVHWFramesContext);
 
             // ffmpeg does not expose RGB vaapi
-            (*hwframe_casted).format = Pixel::VAAPI.into();
-            (*hwframe_casted).sw_format = pixfmt.into();
-            (*hwframe_casted).width = width;
-            (*hwframe_casted).height = height;
-            (*hwframe_casted).initial_pool_size = 5;
+            hwframe_casted.format = self.fmt.into();
+            hwframe_casted.sw_format = pixfmt.into();
+            hwframe_casted.width = width;
+            hwframe_casted.height = height;
+            hwframe_casted.initial_pool_size = 5;
+
+            if self.fmt == Pixel::VULKAN {
+                let vk_ptr = hwframe_casted.hwctx as *mut AVVulkanFramesContext;
+
+                (*vk_ptr).tiling = vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT;
+
+                let mut create_info = vk::ImageDrmFormatModifierListCreateInfoEXT {
+                    drm_format_modifier_count: modifiers.len() as u32,
+                    p_drm_format_modifiers: modifiers.as_ptr() as _,
+                    ..Default::default()
+                };
+                (*vk_ptr).create_pnext = &mut create_info as *mut _ as _;
+            }
 
             let sts = av_hwframe_ctx_init(hwframe);
             if sts != 0 {

src/cap_ext_image_copy.rs

@@ -20,7 +20,7 @@ use wayland_protocols::ext::{
     },
 };
 
-use crate::{CaptureSource, DmabufFormat, DmabufPotentialFormat, DrmModifier, State};
+use crate::{CaptureSource, DmabufFormat, DrmModifier, ReadyCopySource, State};
 
 impl Dispatch<ExtImageCopyCaptureManagerV1, ()> for State<CapExtImageCopy> {
     fn event(
@@ -95,7 +95,7 @@ impl Dispatch<ExtImageCopyCaptureSessionV1, ()> for State<CapExtImageCopy> {
                     if let ExtImageCopyState::Probing(formats, _, _) =
                         &mut state.enc.unwrap_cap().state
                     {
-                        formats.push(DmabufPotentialFormat { fourcc, modifiers })
+                        formats.push(DmabufFormat { fourcc, modifiers })
                     }
                 } else {
                     warn!("Unknown DRM Fourcc: 0x{:08x}", format)
@@ -122,10 +122,10 @@ impl Dispatch<ExtImageCopyCaptureSessionV1, ()> for State<CapExtImageCopy> {
                 }
 
                 let cap = state.enc.unwrap_cap();
-                let (width, height, format, frame) = cap
+                let c = cap
                     .queue_capture_frame(qhandle)
                     .expect("Done without size/format!");
-                state.on_copy_src_ready(width, height, format, qhandle, &frame);
+                state.on_copy_src_ready(c, qhandle);
             }
             ext_image_copy_capture_session_v1::Event::Stopped => {}
             _ => todo!(),
@@ -162,11 +162,7 @@ impl Dispatch<ExtImageCopyCaptureFrameV1, ()> for State<CapExtImageCopy> {
 }
 
 enum ExtImageCopyState {
-    Probing(
-        Vec<DmabufPotentialFormat>,
-        Option<(u32, u32)>,
-        Option<PathBuf>,
-    ),
+    Probing(Vec<DmabufFormat>, Option<(u32, u32)>, Option<PathBuf>),
     Ready(DmabufFormat, (u32, u32)),
 }
 
@@ -217,10 +213,15 @@ impl CaptureSource for CapExtImageCopy {
     fn queue_capture_frame(
         &self,
         eq: &QueueHandle<crate::State<Self>>,
-    ) -> Option<(u32, u32, DrmFourcc, Self::Frame)> {
-        if let ExtImageCopyState::Ready(fmt, (w, h)) = &self.state {
+    ) -> Option<ReadyCopySource<Self::Frame>> {
+        if let ExtImageCopyState::Ready(format, (w, h)) = &self.state {
             let frame = self.output_capture_session.create_frame(eq, ());
-            Some((*w, *h, fmt.fourcc, frame))
+            Some(ReadyCopySource {
+                w: *w,
+                h: *h,
+                format: format.fourcc,
+                frame,
+            })
         } else {
             None
         }

src/cap_wlr_screencopy.rs

@@ -19,7 +19,7 @@ use wayland_protocols_wlr::screencopy::v1::client::{
 };
 
 use crate::{
-    drmGetRenderDeviceNameFromFd, CaptureSource, DmabufPotentialFormat, DrmModifier, State,
+    drmGetRenderDeviceNameFromFd, CaptureSource, DmabufFormat, DrmModifier, ReadyCopySource, State,
 };
 
 impl Dispatch<ZwlrScreencopyManagerV1, ()> for State<CapWlrScreencopy> {
@@ -64,15 +64,23 @@ impl Dispatch<ZwlrScreencopyFrameV1, ()> for State<CapWlrScreencopy> {
                     cap.sent_format = true;
                     let device = cap.drm_device.clone();
                     state.negotiate_format(
-                        &[DmabufPotentialFormat {
+                        &[DmabufFormat {
                             fourcc,
                             modifiers: vec![DrmModifier::LINEAR],
                         }],
                         (dmabuf_width, dmabuf_height),
                         device.as_deref(),
                     );
                 }
-                state.on_copy_src_ready(dmabuf_width, dmabuf_height, fourcc, qhandle, capture);
+                state.on_copy_src_ready(
+                    ReadyCopySource {
+                        w: dmabuf_width,
+                        h: dmabuf_height,
+                        format: fourcc,
+                        frame: capture.clone(),
+                    },
+                    qhandle,
+                );
             }
             zwlr_screencopy_frame_v1::Event::Damage { .. } => {}
             zwlr_screencopy_frame_v1::Event::Buffer { .. } => {}
@@ -168,7 +176,7 @@ impl CaptureSource for CapWlrScreencopy {
     fn queue_capture_frame(
         &self,
         eq: &QueueHandle<State<Self>>,
-    ) -> Option<(u32, u32, DrmFourcc, Self::Frame)> {
+    ) -> Option<ReadyCopySource<Self::Frame>> {
         // creating this triggers the linux_dmabuf event, which is where we allocate etc
 
         let _capture = self