diff --git a/src/platform/linux/pipewire.cpp b/src/platform/linux/pipewire.cpp index 1048b8399ca..d0a267837e5 100644 --- a/src/platform/linux/pipewire.cpp +++ b/src/platform/linux/pipewire.cpp @@ -128,11 +128,17 @@ namespace pipewire { */ struct img_descriptor_t: public egl::img_descriptor_t { ~img_descriptor_t() override { - if (data) { + // Only free buffers this image actually owns. The memory-buffer capture + // path points img->data at the PipeWire staging vector (front_buffer), + // which is owned by pipewire_t -- deleting it here corrupts the heap. + if (data && data_owned) { delete[] data; - data = nullptr; } + data = nullptr; + data_owned = false; } + + bool data_owned = false; ///< Whether img->data is owned by this image and must be freed. }; /** @@ -430,12 +436,13 @@ namespace pipewire { struct spa_buffer *buf = stream_data.current_buffer->buffer; if (buf->datas[0].chunk->size != 0) { - auto *img_descriptor = static_cast(img); + auto *img_descriptor = static_cast(img); fill_img_metadata(img_descriptor, buf); if (buf->datas[0].type == SPA_DATA_DmaBuf) { fill_img_dmabuf(img_descriptor, buf, stream_data); } else { img->data = stream_data.front_buffer->data(); + img_descriptor->data_owned = false; img->row_pitch = stream_data.local_stride; } } @@ -937,6 +944,7 @@ namespace pipewire { img->sequence = 0; img->serial = std::numeric_limitsserial)>::max(); img->data = nullptr; + img->data_owned = false; std::fill_n(img->sd.fds, 4, -1); return img; @@ -1061,7 +1069,20 @@ namespace pipewire { * @return Capture status reported to the streaming pipeline. */ int dummy_img(platf::img_t *img) override { - // Empty images are recognized as dummies by the zero sequence number + // Software encoders convert the dummy image immediately; provide a valid + // (black) buffer instead of leaving img->data null, which makes sws fail + // with EINVAL. The buffer is new[]-allocated and marked as owned so the + // destructor releases it. + if (img->data == nullptr) { + const auto w = img->width; + const auto h = img->height; + if (w > 0 && h > 0) { + img->data = new uint8_t[static_cast(w) * h * 4](); // NOSONAR(cpp:S5025) - buffer is owned by the image and freed by img_descriptor_t's destructor + static_cast(img)->data_owned = true; + img->row_pitch = w * 4; + img->pixel_pitch = 4; + } + } return 0; } diff --git a/src/video.cpp b/src/video.cpp index 5306f37d706..e58ed359e7c 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -209,9 +209,34 @@ namespace video { // If we need to add aspect ratio padding, we need to scale into an intermediate output buffer bool requires_padding = (sw_frame->width != sws_output_frame->width || sw_frame->height != sws_output_frame->height); + // Detect the actual capture pixel format. PipeWire-based captures (KWin + // screencast / XDG portal) deliver NV12 with a 1-byte-per-pixel row + // pitch, while KMS/DMABUF captures deliver BGR0 (4 bytes per pixel). + const auto input_fmt = (img.row_pitch == img.width) ? AV_PIX_FMT_NV12 : AV_PIX_FMT_BGR0; + + // The sws context is created with the default BGR0 source format; + // recreate it once if the capture is actually NV12. + if (input_fmt != sws_src_format) { + sws_src_format = input_fmt; + if (reinit_sws(input_fmt) < 0) { + return -1; + } + } + // Setup the input frame using the caller's img_t sws_input_frame->data[0] = img.data; sws_input_frame->linesize[0] = img.row_pitch; + if (input_fmt == AV_PIX_FMT_NV12) { + sws_input_frame->data[1] = img.data + static_cast(img.row_pitch) * img.height; + sws_input_frame->linesize[1] = img.row_pitch; + } else { + sws_input_frame->data[1] = nullptr; + sws_input_frame->linesize[1] = 0; + } + sws_input_frame->data[2] = nullptr; + sws_input_frame->linesize[2] = 0; + sws_input_frame->data[3] = nullptr; + sws_input_frame->linesize[3] = 0; // Perform color conversion and scaling to the final size auto status = sws_scale_frame(sws.get(), requires_padding ? sws_output_frame.get() : sw_frame.get(), sws_input_frame.get()); @@ -341,6 +366,20 @@ namespace video { offsetW = (frame->width - out_width) / 2; offsetH = (frame->height - out_height) / 2; + sws_src_format = AV_PIX_FMT_BGR0; + + return reinit_sws(sws_src_format); + } + + /** + * @brief (Re)create the software scaler for the given source format. + * + * @param src_format Pixel format of the captured frames. + * @return 0 on success; nonzero on failure. + */ + int reinit_sws(AVPixelFormat src_format) { + sws_input_frame->format = src_format; + sws.reset(sws_alloc_context()); if (!sws) { return -1; @@ -349,7 +388,7 @@ namespace video { AVDictionary *options {nullptr}; av_dict_set_int(&options, "srcw", sws_input_frame->width, 0); av_dict_set_int(&options, "srch", sws_input_frame->height, 0); - av_dict_set_int(&options, "src_format", sws_input_frame->format, 0); + av_dict_set_int(&options, "src_format", src_format, 0); av_dict_set_int(&options, "dstw", sws_output_frame->width, 0); av_dict_set_int(&options, "dsth", sws_output_frame->height, 0); av_dict_set_int(&options, "dst_format", sws_output_frame->format, 0); @@ -381,6 +420,7 @@ namespace video { avcodec_frame_t sws_input_frame; ///< Sws input frame. avcodec_frame_t sws_output_frame; ///< Sws output frame. sws_t sws; ///< Software scaler used when frames need CPU-side pixel conversion. + AVPixelFormat sws_src_format {AV_PIX_FMT_BGR0}; ///< Source format the sws context was created with. // Offset of input image to output frame in pixels int offsetW; ///< Offset w. diff --git a/tests/unit/test_video.cpp b/tests/unit/test_video.cpp index 6ad2d3a83a0..5b0973df538 100644 --- a/tests/unit/test_video.cpp +++ b/tests/unit/test_video.cpp @@ -9,6 +9,13 @@ #include #include #include +#include + +// ffmpeg includes +extern "C" { +#include +#include +} // local includes #include @@ -167,3 +174,67 @@ INSTANTIATE_TEST_SUITE_P( std::make_tuple(120, 11988, std::chrono::nanoseconds {8341666}) // 1e9 * 1001 / 120000 ) ); + +namespace video { + // Redeclaration of the internal software encoder device so the test can + // exercise conversion without a real display. The member layout must match + // the definition in src/video.cpp (compiled into this test binary); keep it + // in sync if the class changes. + class avcodec_software_encode_device_t: public platf::avcodec_encode_device_t { + public: + int init(int in_width, int in_height, AVFrame *frame, AVPixelFormat format, bool hardware); + int set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override; + int convert(platf::img_t &img) override; + + private: + avcodec_frame_t hw_frame; ///< Hw frame. + avcodec_frame_t sw_frame; ///< Sw frame. + avcodec_frame_t sws_input_frame; ///< Sws input frame. + avcodec_frame_t sws_output_frame; ///< Sws output frame. + sws_t sws; ///< Software scaler used when frames need CPU-side pixel conversion. + AVPixelFormat sws_src_format {AV_PIX_FMT_BGR0}; ///< Source format the sws context was created with. + int offsetW; ///< Offset w. + int offsetH; ///< Offset h. + }; +} // namespace video + +/** + * @brief Software encoder converts both BGR0 and NV12 captured frames. + */ +TEST(SoftwareEncoderConversion, Bgr0AndNv12) { + constexpr int w = 320; + constexpr int h = 240; + + AVFrame *frame = av_frame_alloc(); + ASSERT_NE(frame, nullptr); + frame->width = w; + frame->height = h; + frame->format = AV_PIX_FMT_YUV420P; + + video::avcodec_software_encode_device_t device; + ASSERT_EQ(device.init(w, h, frame, AV_PIX_FMT_YUV420P, false), 0); + ASSERT_EQ(device.set_frame(frame, nullptr), 0); + + // BGR0 frame (4 bytes per pixel) -- the classic KMS/DMABUF capture layout. + std::vector bgr0_buffer(static_cast(w) * h * 4); + platf::img_t bgr0_img {}; + bgr0_img.data = bgr0_buffer.data(); + bgr0_img.width = w; + bgr0_img.height = h; + bgr0_img.row_pitch = w * 4; + bgr0_img.pixel_pitch = 4; + EXPECT_EQ(device.convert(bgr0_img), 0); + + // NV12 frame (1 byte per pixel row pitch, Y plane + interleaved UV) -- the + // layout delivered by PipeWire-based captures (KWin screencast / portal). + std::vector nv12_buffer(static_cast(w) * h + static_cast(w) * h / 2); + platf::img_t nv12_img {}; + nv12_img.data = nv12_buffer.data(); + nv12_img.width = w; + nv12_img.height = h; + nv12_img.row_pitch = w; + nv12_img.pixel_pitch = 1; + EXPECT_EQ(device.convert(nv12_img), 0); + + av_frame_free(&frame); +}