fix(linux/pipewire): restore dummy buffer for software encoders and fix buffer ownership

src/platform/linux/pipewire.cpp

@@ -128,11 +128,17 @@ namespace pipewire {
    */
   struct img_descriptor_t: public egl::img_descriptor_t {
     ~img_descriptor_t() override {
-      if (data) {
+      // Only free buffers this image actually owns. The memory-buffer capture
+      // path points img->data at the PipeWire staging vector (front_buffer),
+      // which is owned by pipewire_t -- deleting it here corrupts the heap.
+      if (data && data_owned) {
         delete[] data;
-        data = nullptr;
       }
+      data = nullptr;
+      data_owned = false;
     }
+
+    bool data_owned = false;  ///< Whether img->data is owned by this image and must be freed.
   };
 
   /**
@@ -430,12 +436,13 @@ namespace pipewire {
 
       struct spa_buffer *buf = stream_data.current_buffer->buffer;
       if (buf->datas[0].chunk->size != 0) {
-        auto *img_descriptor = static_cast<egl::img_descriptor_t *>(img);
+        auto *img_descriptor = static_cast<img_descriptor_t *>(img);
         fill_img_metadata(img_descriptor, buf);
         if (buf->datas[0].type == SPA_DATA_DmaBuf) {
           fill_img_dmabuf(img_descriptor, buf, stream_data);
         } else {
           img->data = stream_data.front_buffer->data();
+          img_descriptor->data_owned = false;
           img->row_pitch = stream_data.local_stride;
         }
       }
@@ -937,6 +944,7 @@ namespace pipewire {
       img->sequence = 0;
       img->serial = std::numeric_limits<decltype(img->serial)>::max();
       img->data = nullptr;
+      img->data_owned = false;
       std::fill_n(img->sd.fds, 4, -1);
 
       return img;
@@ -1061,7 +1069,20 @@ namespace pipewire {
      * @return Capture status reported to the streaming pipeline.
      */
     int dummy_img(platf::img_t *img) override {
-      // Empty images are recognized as dummies by the zero sequence number
+      // Software encoders convert the dummy image immediately; provide a valid
+      // (black) buffer instead of leaving img->data null, which makes sws fail
+      // with EINVAL. The buffer is new[]-allocated and marked as owned so the
+      // destructor releases it.
+      if (img->data == nullptr) {
+        const auto w = img->width;
+        const auto h = img->height;
+        if (w > 0 && h > 0) {
+          img->data = new uint8_t[static_cast<size_t>(w) * h * 4]();  // NOSONAR(cpp:S5025) - buffer is owned by the image and freed by img_descriptor_t's destructor
+          static_cast<img_descriptor_t *>(img)->data_owned = true;
+          img->row_pitch = w * 4;
+          img->pixel_pitch = 4;
+        }
+      }
       return 0;
     }
 

src/video.cpp

@@ -209,9 +209,34 @@ namespace video {
       // If we need to add aspect ratio padding, we need to scale into an intermediate output buffer
       bool requires_padding = (sw_frame->width != sws_output_frame->width || sw_frame->height != sws_output_frame->height);
 
+      // Detect the actual capture pixel format. PipeWire-based captures (KWin
+      // screencast / XDG portal) deliver NV12 with a 1-byte-per-pixel row
+      // pitch, while KMS/DMABUF captures deliver BGR0 (4 bytes per pixel).
+      const auto input_fmt = (img.row_pitch == img.width) ? AV_PIX_FMT_NV12 : AV_PIX_FMT_BGR0;
+
+      // The sws context is created with the default BGR0 source format;
+      // recreate it once if the capture is actually NV12.
+      if (input_fmt != sws_src_format) {
+        sws_src_format = input_fmt;
+        if (reinit_sws(input_fmt) < 0) {
+          return -1;
+        }
+      }
+
       // Setup the input frame using the caller's img_t
       sws_input_frame->data[0] = img.data;
       sws_input_frame->linesize[0] = img.row_pitch;
+      if (input_fmt == AV_PIX_FMT_NV12) {
+        sws_input_frame->data[1] = img.data + static_cast<std::size_t>(img.row_pitch) * img.height;
+        sws_input_frame->linesize[1] = img.row_pitch;
+      } else {
+        sws_input_frame->data[1] = nullptr;
+        sws_input_frame->linesize[1] = 0;
+      }
+      sws_input_frame->data[2] = nullptr;
+      sws_input_frame->linesize[2] = 0;
+      sws_input_frame->data[3] = nullptr;
+      sws_input_frame->linesize[3] = 0;
 
       // Perform color conversion and scaling to the final size
       auto status = sws_scale_frame(sws.get(), requires_padding ? sws_output_frame.get() : sw_frame.get(), sws_input_frame.get());
@@ -341,6 +366,20 @@ namespace video {
       offsetW = (frame->width - out_width) / 2;
       offsetH = (frame->height - out_height) / 2;
 
+      sws_src_format = AV_PIX_FMT_BGR0;
+
+      return reinit_sws(sws_src_format);
+    }
+
+    /**
+     * @brief (Re)create the software scaler for the given source format.
+     *
+     * @param src_format Pixel format of the captured frames.
+     * @return 0 on success; nonzero on failure.
+     */
+    int reinit_sws(AVPixelFormat src_format) {
+      sws_input_frame->format = src_format;
+
       sws.reset(sws_alloc_context());
       if (!sws) {
         return -1;
@@ -349,7 +388,7 @@ namespace video {
       AVDictionary *options {nullptr};
       av_dict_set_int(&options, "srcw", sws_input_frame->width, 0);
       av_dict_set_int(&options, "srch", sws_input_frame->height, 0);
-      av_dict_set_int(&options, "src_format", sws_input_frame->format, 0);
+      av_dict_set_int(&options, "src_format", src_format, 0);
       av_dict_set_int(&options, "dstw", sws_output_frame->width, 0);
       av_dict_set_int(&options, "dsth", sws_output_frame->height, 0);
       av_dict_set_int(&options, "dst_format", sws_output_frame->format, 0);
@@ -381,6 +420,7 @@ namespace video {
     avcodec_frame_t sws_input_frame;  ///< Sws input frame.
     avcodec_frame_t sws_output_frame;  ///< Sws output frame.
     sws_t sws;  ///< Software scaler used when frames need CPU-side pixel conversion.
+    AVPixelFormat sws_src_format {AV_PIX_FMT_BGR0};  ///< Source format the sws context was created with.
 
     // Offset of input image to output frame in pixels
     int offsetW;  ///< Offset w.

tests/unit/test_video.cpp

@@ -9,6 +9,13 @@
 #include <algorithm>
 #include <tuple>
 #include <utility>
+#include <vector>
+
+// ffmpeg includes
+extern "C" {
+#include <libavutil/frame.h>
+#include <libavutil/pixfmt.h>
+}
 
 // local includes
 #include <src/config.h>
@@ -167,3 +174,67 @@ INSTANTIATE_TEST_SUITE_P(
     std::make_tuple(120, 11988, std::chrono::nanoseconds {8341666})  // 1e9 * 1001 / 120000
   )
 );
+
+namespace video {
+  // Redeclaration of the internal software encoder device so the test can
+  // exercise conversion without a real display. The member layout must match
+  // the definition in src/video.cpp (compiled into this test binary); keep it
+  // in sync if the class changes.
+  class avcodec_software_encode_device_t: public platf::avcodec_encode_device_t {
+  public:
+    int init(int in_width, int in_height, AVFrame *frame, AVPixelFormat format, bool hardware);
+    int set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override;
+    int convert(platf::img_t &img) override;
+
+  private:
+    avcodec_frame_t hw_frame;  ///< Hw frame.
+    avcodec_frame_t sw_frame;  ///< Sw frame.
+    avcodec_frame_t sws_input_frame;  ///< Sws input frame.
+    avcodec_frame_t sws_output_frame;  ///< Sws output frame.
+    sws_t sws;  ///< Software scaler used when frames need CPU-side pixel conversion.
+    AVPixelFormat sws_src_format {AV_PIX_FMT_BGR0};  ///< Source format the sws context was created with.
+    int offsetW;  ///< Offset w.
+    int offsetH;  ///< Offset h.
+  };
+}  // namespace video
+
+/**
+ * @brief Software encoder converts both BGR0 and NV12 captured frames.
+ */
+TEST(SoftwareEncoderConversion, Bgr0AndNv12) {
+  constexpr int w = 320;
+  constexpr int h = 240;
+
+  AVFrame *frame = av_frame_alloc();
+  ASSERT_NE(frame, nullptr);
+  frame->width = w;
+  frame->height = h;
+  frame->format = AV_PIX_FMT_YUV420P;
+
+  video::avcodec_software_encode_device_t device;
+  ASSERT_EQ(device.init(w, h, frame, AV_PIX_FMT_YUV420P, false), 0);
+  ASSERT_EQ(device.set_frame(frame, nullptr), 0);
+
+  // BGR0 frame (4 bytes per pixel) -- the classic KMS/DMABUF capture layout.
+  std::vector<uint8_t> bgr0_buffer(static_cast<size_t>(w) * h * 4);
+  platf::img_t bgr0_img {};
+  bgr0_img.data = bgr0_buffer.data();
+  bgr0_img.width = w;
+  bgr0_img.height = h;
+  bgr0_img.row_pitch = w * 4;
+  bgr0_img.pixel_pitch = 4;
+  EXPECT_EQ(device.convert(bgr0_img), 0);
+
+  // NV12 frame (1 byte per pixel row pitch, Y plane + interleaved UV) -- the
+  // layout delivered by PipeWire-based captures (KWin screencast / portal).
+  std::vector<uint8_t> nv12_buffer(static_cast<size_t>(w) * h + static_cast<size_t>(w) * h / 2);
+  platf::img_t nv12_img {};
+  nv12_img.data = nv12_buffer.data();
+  nv12_img.width = w;
+  nv12_img.height = h;
+  nv12_img.row_pitch = w;
+  nv12_img.pixel_pitch = 1;
+  EXPECT_EQ(device.convert(nv12_img), 0);
+
+  av_frame_free(&frame);
+}