fix(linux): skip h264_vulkan on RADV Raven, whose takes come out corrupt

electron/native/pipewire-capture/src/encoder.rs

@@ -269,6 +269,73 @@ unsafe fn vulkan_can_encode_h264(device: *mut ff::AVBufferRef) -> bool {
     })
 }
 
+/// True for a Vulkan device whose H.264 encode opens but writes a broken file.
+///
+/// RADV on the Raven family (Raven, Picasso, Raven2: VCN 1.0) advertises an
+/// H.264 encode queue once `video_encode` is enabled, and `h264_vulkan` opens on
+/// it, but the take comes out with blocks of older content stuck in the picture
+/// and few distinct frames (issue #1465, a Ryzen 5 3550H). libopenh264 records
+/// the same screen cleanly, so the ladder must fall through to it.
+///
+/// Mesa names these devices "... (RADV RAVEN)" and "... (RADV RAVEN2)".
+// ponytail: matched by name on one report; widen to other VCN generations only
+// if a take from one shows the same corruption.
+fn vulkan_encode_is_broken_on(device_name: &str) -> bool {
+    device_name.contains("(RADV RAVEN")
+}
+
+/// The `deviceName` of the physical device behind `device`, read the same way
+/// as [`vulkan_can_encode_h264`]: through Vulkan, from the first four fields of
+/// `AVVulkanDeviceContext`. Empty if Vulkan does not expose the query.
+///
+/// SAFETY: `device` is a live AV_HWDEVICE_TYPE_VULKAN device context.
+unsafe fn vulkan_device_name(device: *mut ff::AVBufferRef) -> String {
+    use std::ffi::c_void;
+
+    #[repr(C)]
+    struct DeviceHead {
+        alloc: *const c_void,
+        get_proc_addr: Option<GetInstanceProcAddr>,
+        instance: *mut c_void,
+        physical_device: *mut c_void,
+    }
+    /// `VkPhysicalDeviceProperties`: the fields up to `deviceName`, then room for
+    /// the rest (824 bytes in all on 64-bit), which is not read.
+    #[repr(C, align(8))]
+    struct Properties {
+        api_version: u32,
+        driver_version: u32,
+        vendor_id: u32,
+        device_id: u32,
+        device_type: i32,
+        device_name: [std::ffi::c_char; 256],
+        rest: [u8; 1024],
+    }
+    type GetInstanceProcAddr =
+        unsafe extern "C" fn(*mut c_void, *const std::ffi::c_char) -> *const c_void;
+    type GetProperties = unsafe extern "C" fn(*mut c_void, *mut Properties);
+
+    let context = (*device).data as *const ff::AVHWDeviceContext;
+    let head = (*context).hwctx as *const DeviceHead;
+    let Some(get_proc_addr) = (*head).get_proc_addr else {
+        return String::new();
+    };
+    let query = get_proc_addr((*head).instance, c"vkGetPhysicalDeviceProperties".as_ptr());
+    if query.is_null() {
+        return String::new();
+    }
+    let query: GetProperties = std::mem::transmute(query);
+    let mut properties: Properties = std::mem::zeroed();
+    query((*head).physical_device, &mut properties);
+    // The zeroed tail guarantees a terminator even from a driver that filled all
+    // 256 bytes of the name.
+    let last = properties.device_name.len() - 1;
+    properties.device_name[last] = 0;
+    CStr::from_ptr(properties.device_name.as_ptr())
+        .to_string_lossy()
+        .into_owned()
+}
+
 const RTLD_LAZY: i32 = 1;
 const RTLD_LOCAL: i32 = 0;
 
@@ -592,6 +659,14 @@ impl VideoEncoder {
         if device_type == ff::AV_HWDEVICE_TYPE_VULKAN && !vulkan_can_encode_h264(self.hw_device) {
             return Err("the Vulkan device has no H.264 encode queue".to_owned());
         }
+        if device_type == ff::AV_HWDEVICE_TYPE_VULKAN {
+            let name = vulkan_device_name(self.hw_device);
+            if vulkan_encode_is_broken_on(&name) {
+                return Err(format!(
+                    "h264_vulkan corrupts its output on {name} (VCN 1.0, issue #1465)"
+                ));
+            }
+        }
 
         self.hw_frames = ff::av_hwframe_ctx_alloc(self.hw_device);
         if self.hw_frames.is_null() {
@@ -1772,6 +1847,16 @@ mod tests {
         let _ = std::fs::remove_file(&output);
     }
 
+    #[test]
+    fn vulkan_is_skipped_on_vcn1_radv_only() {
+        assert!(vulkan_encode_is_broken_on("AMD Radeon Vega 8 Graphics (RADV RAVEN)"));
+        assert!(vulkan_encode_is_broken_on("AMD Radeon Vega 3 Graphics (RADV RAVEN2)"));
+        assert!(!vulkan_encode_is_broken_on("AMD Radeon Graphics (RADV RENOIR)"));
+        assert!(!vulkan_encode_is_broken_on("AMD Radeon RX 7600 (RADV NAVI33)"));
+        assert!(!vulkan_encode_is_broken_on("NVIDIA GeForce GTX 1650"));
+        assert!(!vulkan_encode_is_broken_on(""));
+    }
+
     #[test]
     fn radv_flags_are_added_without_clobbering_an_existing_value() {
         std::env::set_var("RADV_PERFTEST", "gpl");