From 93d65fce7fc9a038f02af181f47b1988abaf7d85 Mon Sep 17 00:00:00 2001 From: jw098 Date: Wed, 21 Jan 2026 22:42:55 -0800 Subject: [PATCH 01/20] Stream History: switch back to using SaveFrames.h --- .../Recording/StreamHistorySession.cpp | 6 ++-- .../StreamHistoryTracker_SaveFrames.h | 32 ++++++++++++------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp index f7dc856aa0..95faa7b6fe 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp @@ -13,11 +13,11 @@ #include "CommonFramework/Recording/StreamHistoryOption.h" #if (QT_VERSION_MAJOR == 6) && (QT_VERSION_MINOR >= 8) -//#include "StreamHistoryTracker_SaveFrames.h" +#include "StreamHistoryTracker_SaveFrames.h" //#include "StreamHistoryTracker_RecordOnTheFly.h" -#include "StreamHistoryTracker_ParallelStreams.h" +// #include "StreamHistoryTracker_ParallelStreams.h" #else -#include "StreamHistoryTracker_Null.h" +// #include "StreamHistoryTracker_Null.h" #endif diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 433209e12c..45ecb9dabd 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -49,22 +49,25 @@ struct AudioBlock{ class StreamHistoryTracker{ public: StreamHistoryTracker( + Logger& logger, + std::chrono::seconds window, size_t audio_samples_per_frame, size_t audio_frames_per_second, - std::chrono::seconds window + bool has_video ); void set_window(std::chrono::seconds window); - bool save(Logger& logger, const std::string& filename) const; + bool save(const std::string& filename) const; public: void on_samples(const float* data, size_t frames); - void on_frame(std::shared_ptr frame); + void on_frame(std::shared_ptr frame); private: void clear_old(); private: + Logger& m_logger; mutable SpinLock m_lock; std::chrono::seconds m_window; @@ -72,26 +75,31 @@ class StreamHistoryTracker{ const size_t m_audio_frames_per_second; const size_t m_audio_samples_per_second; const double m_microseconds_per_sample; + const bool m_has_video; // We use shared_ptr here so it's fast to snapshot when we need to copy // everything asynchronously. std::deque> m_audio; - std::deque> m_frames; + std::deque> m_frames; }; StreamHistoryTracker::StreamHistoryTracker( + Logger& logger, + std::chrono::seconds window, size_t audio_samples_per_frame, size_t audio_frames_per_second, - std::chrono::seconds window + bool has_video ) - : m_window(window) + : m_logger(logger) + , m_window(window) , m_audio_samples_per_frame(audio_samples_per_frame) , m_audio_frames_per_second(audio_frames_per_second) , m_audio_samples_per_second(audio_samples_per_frame * audio_frames_per_second) , m_microseconds_per_sample(1. / (m_audio_samples_per_second * 1000000.)) + , m_has_video(has_video) {} void StreamHistoryTracker::set_window(std::chrono::seconds window){ @@ -111,7 +119,7 @@ void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ )); clear_old(); } -void StreamHistoryTracker::on_frame(std::shared_ptr frame){ +void StreamHistoryTracker::on_frame(std::shared_ptr frame){ // TODO: Find a more efficient way to buffer the frames. // It takes almost 10GB of memory to store 30 seconds of QVideoFrames // due to them caching uncompressed bitmaps. @@ -162,11 +170,11 @@ void StreamHistoryTracker::clear_old(){ -bool StreamHistoryTracker::save(Logger& logger, const std::string& filename) const{ - logger.log("Saving stream history...", COLOR_BLUE); +bool StreamHistoryTracker::save(const std::string& filename) const{ + m_logger.log("Saving stream history...", COLOR_BLUE); std::deque> audio; - std::deque> frames; + std::deque> frames; { // Fast copy the current state of the stream. WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); @@ -285,7 +293,7 @@ bool StreamHistoryTracker::save(Logger& logger, const std::string& filename) con #endif if (current_time() - last_change > std::chrono::seconds(10)){ - logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); + m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); success = false; break; } @@ -294,7 +302,7 @@ bool StreamHistoryTracker::save(Logger& logger, const std::string& filename) con } recorder.stop(); - logger.log("Done saving stream history...", COLOR_BLUE); + m_logger.log("Done saving stream history...", COLOR_BLUE); // cout << recorder.duration() << endl; From fab31678963e0822db29c6e1adb51c09ebae518c Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 22 Jan 2026 22:57:09 -0800 Subject: [PATCH 02/20] draft: save compressed frames. --- .../StreamHistoryTracker_SaveFrames.h | 91 ++++++++++++++++--- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 45ecb9dabd..8282315ee7 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -10,6 +10,7 @@ #ifndef PokemonAutomation_StreamHistoryTracker_SaveFrames_H #define PokemonAutomation_StreamHistoryTracker_SaveFrames_H +#include #include #include #include @@ -19,14 +20,15 @@ #include #include #include +#include #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Concurrency/SpinLock.h" #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" -//#include -//using std::cout; -//using std::endl; +#include +using std::cout; +using std::endl; namespace PokemonAutomation{ @@ -45,6 +47,11 @@ struct AudioBlock{ {} }; +struct CompressedVideoFrame{ + WallClock timestamp; + std::vector compressed_frame; +}; + class StreamHistoryTracker{ public: @@ -81,6 +88,7 @@ class StreamHistoryTracker{ // everything asynchronously. std::deque> m_audio; std::deque> m_frames; + std::deque m_compressed_frames; }; @@ -119,6 +127,59 @@ void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ )); clear_old(); } + +std::vector compress_video_frame(const QVideoFrame& const_frame) { + // Create a local non-const copy (cheap, uses explicit sharing) + QVideoFrame frame = const_frame; + + // 1. Map the frame to CPU memory + if (!frame.map(QVideoFrame::ReadOnly)) { + return {}; + } + + // Ensure unmap() is called when this function exits (success or failure) + auto guard = qScopeGuard([&frame] { frame.unmap(); }); + + // 2. Convert to QImage (Qt 6.8+ handles internal conversions efficiently) + // For circular buffers, using a 3-channel RGB888 is common for OpenCV + QImage img = frame.toImage().convertToFormat(QImage::Format_RGB888); + + // 3. Wrap QImage memory into a cv::Mat (No-copy) + // Note: OpenCV expects BGR by default, but QImage is RGB. + // If color accuracy matters, use cv::cvtColor later or img.rgbSwapped(). + cv::Mat mat(img.height(), img.width(), CV_8UC3, + const_cast(img.bits()), img.bytesPerLine()); + + // 4. Compress using imencode + std::vector compressed_buffer; + std::vector params = {cv::IMWRITE_JPEG_QUALITY, 80}; // 0-100 + + // Convert RGB to BGR before encoding because imencode expects BGR + cv::Mat bgr_Mat; + cv::cvtColor(mat, bgr_Mat, cv::COLOR_RGB2BGR); + + cv::imencode(".jpg", bgr_Mat, compressed_buffer, params); + + return compressed_buffer; // Store this in your circular buffer +} + +QVideoFrame decompress_video_frame(const std::vector &compressed_buffer) { + if (compressed_buffer.empty()) return {}; + + // 1. Decompress JPEG buffer into a QImage + // fromData handles the JPEG header and decompression automatically + QImage img = QImage::fromData(compressed_buffer.data(), + static_cast(compressed_buffer.size()), + "JPG"); + + if (img.isNull()) return {}; + + // 2. Use the new Qt 6.8 constructor + // This wraps the QImage into a QVideoFrame efficiently. + // If the format is compatible (like RGB888), it minimizes copies. + return QVideoFrame(img); +} + void StreamHistoryTracker::on_frame(std::shared_ptr frame){ // TODO: Find a more efficient way to buffer the frames. // It takes almost 10GB of memory to store 30 seconds of QVideoFrames @@ -127,7 +188,9 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); // cout << "on_frame() = " << m_frames.size() << endl; - m_frames.emplace_back(std::move(frame)); + auto compressed_frame = compress_video_frame(frame->frame); + m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, compressed_frame}); + // m_frames.emplace_back(std::move(frame)); clear_old(); } @@ -149,7 +212,7 @@ void StreamHistoryTracker::clear_old(){ static_cast((double)block.samples.size() * m_microseconds_per_sample) ); - if (end_block < threshold){ + if (end_block < threshold){ // todo: confirm if the audio deque clears properly m_audio.pop_front(); }else{ break; @@ -157,9 +220,15 @@ void StreamHistoryTracker::clear_old(){ } // cout << "exit" << endl; - while (!m_frames.empty()){ - if (m_frames.front()->timestamp < threshold){ - m_frames.pop_front(); + while (!m_compressed_frames.empty()){ + // if (m_frames.front()->timestamp < threshold){ + // m_frames.pop_front(); + // }else{ + // break; + // } + + if (m_compressed_frames.front().timestamp < threshold){ + m_compressed_frames.pop_front(); }else{ break; } @@ -174,15 +243,15 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ m_logger.log("Saving stream history...", COLOR_BLUE); std::deque> audio; - std::deque> frames; + std::deque frames; { // Fast copy the current state of the stream. WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_audio.empty() && m_frames.empty()){ + if (m_audio.empty() && m_compressed_frames.empty()){ return false; } audio = m_audio; - frames = m_frames; + frames = m_compressed_frames; } // Now that the lock is released, we can take our time encoding it. From 6a5683c69efd18379594931b3fb5329b263d3425 Mon Sep 17 00:00:00 2001 From: jw098 Date: Sun, 25 Jan 2026 16:46:16 -0800 Subject: [PATCH 03/20] fix build --- .../Recording/StreamHistoryTracker_SaveFrames.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 8282315ee7..ee6ea1945f 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -243,15 +243,15 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ m_logger.log("Saving stream history...", COLOR_BLUE); std::deque> audio; - std::deque frames; + std::deque> frames; { // Fast copy the current state of the stream. WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_audio.empty() && m_compressed_frames.empty()){ + if (m_audio.empty() && m_frames.empty()){ return false; } audio = m_audio; - frames = m_compressed_frames; + frames = m_frames; } // Now that the lock is released, we can take our time encoding it. From f55f6389d9fdb20dd7aa0e62b1e07bbeadfc0962 Mon Sep 17 00:00:00 2001 From: jw098 Date: Mon, 26 Jan 2026 13:03:36 -0800 Subject: [PATCH 04/20] split StreamHistory tracker into .h and .cpp file. add draft VideoGenerator class. --- .../StreamHistoryTracker_SaveFrames.cpp | 482 ++++++++++++++++++ .../StreamHistoryTracker_SaveFrames.h | 323 +----------- SerialPrograms/cmake/SourceFiles.cmake | 1 + 3 files changed, 501 insertions(+), 305 deletions(-) create mode 100644 SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp new file mode 100644 index 0000000000..3d7328ccb2 --- /dev/null +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -0,0 +1,482 @@ +/* Stream History Tracker + * + * From: https://github.com/PokemonAutomation/ + * + * Implement by saving the last X seconds of frames. This is currently not + * viable because the QVideoFrames are uncompressed. + * + */ + +#include +// #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Logging/AbstractLogger.h" +#include "Common/Cpp/Concurrency/SpinLock.h" +#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" +#include "StreamHistoryTracker_SaveFrames.h" + +#include +using std::cout; +using std::endl; + + +namespace PokemonAutomation{ + + + +QVideoFrame decompress_video_frame(const std::vector &compressed_buffer) { + if (compressed_buffer.empty()) return {}; + + // 1. Decompress JPEG buffer into a QImage + // fromData handles the JPEG header and decompression automatically + QImage img = QImage::fromData(compressed_buffer.data(), + static_cast(compressed_buffer.size()), + "JPG"); + + if (img.isNull()) return {}; + + // 2. Use the new Qt 6.8 constructor + // This wraps the QImage into a QVideoFrame efficiently. + // If the format is compatible (like RGB888), it minimizes copies. + return QVideoFrame(img); +} + +std::vector compress_video_frame(const QVideoFrame& const_frame) { + // Create a local non-const copy (cheap, uses explicit sharing) + QVideoFrame frame = const_frame; + + // 1. Map the frame to CPU memory + if (!frame.map(QVideoFrame::ReadOnly)) { + return {}; + } + + // Ensure unmap() is called when this function exits (success or failure) + auto guard = qScopeGuard([&frame] { frame.unmap(); }); + + // 2. Convert to QImage (Qt 6.8+ handles internal conversions efficiently) + // For circular buffers, using a 3-channel RGB888 is common for OpenCV + QImage img = frame.toImage().convertToFormat(QImage::Format_RGB888); + + // 3. Wrap QImage memory into a cv::Mat (No-copy) + // Note: OpenCV expects BGR by default, but QImage is RGB. + // If color accuracy matters, use cv::cvtColor later or img.rgbSwapped(). + cv::Mat mat(img.height(), img.width(), CV_8UC3, + const_cast(img.bits()), img.bytesPerLine()); + + // 4. Compress using imencode + std::vector compressed_buffer; + std::vector params = {cv::IMWRITE_JPEG_QUALITY, 80}; // 0-100 + + // Convert RGB to BGR before encoding because imencode expects BGR + cv::Mat bgr_Mat; + cv::cvtColor(mat, bgr_Mat, cv::COLOR_RGB2BGR); + + cv::imencode(".jpg", bgr_Mat, compressed_buffer, params); + + return compressed_buffer; // Store this in your circular buffer +} + + +class VideoGenerator : public QObject { + Q_OBJECT +public: + VideoGenerator(QVideoFrameInput *input, std::deque frames) + : m_input(input), m_frames(frames){ + + // Listen for the signal that the recorder's buffer has space + connect(m_input, &QVideoFrameInput::readyToSendVideoFrame, this, &VideoGenerator::sendNextFrame); + } + +public slots: + void sendNextFrame() { + // Send frames in a loop until the buffer is full or we run out of images + if (!m_frames.empty()) + { + QVideoFrame frame = decompress_video_frame(m_frames.front().compressed_frame); // m_frameQueue.dequeue(); + + // Set timestamp (e.g., 30 FPS = 33333 microseconds per frame) + // frame.setStartTime(m_index * 33333); + + // 2. ONLY send the frame if the input is ready + bool success = m_input->sendVideoFrame(frame); + + // 3. Optional: Check if the frame was dropped (shouldn't happen + // often if you are responding to readyToSendVideoFrame) + if (!success) { + cout << "Frame was rejected, stopping frame emission." << endl;; + // You might re-enqueue the frame or just discard it based on your needs. + } + } + else { + cout << "Queue empty, waiting for more frames..."; + // Finalize: Sending an empty frame signals the end of the stream + m_input->sendVideoFrame(QVideoFrame()); + emit finished(); + } + + } + +signals: + void finished(); + +private: + QVideoFrameInput *m_input; + std::deque m_frames; +}; + + +StreamHistoryTracker::StreamHistoryTracker( + Logger& logger, + std::chrono::seconds window, + size_t audio_samples_per_frame, + size_t audio_frames_per_second, + bool has_video +) + : m_logger(logger) + , m_window(window) + , m_audio_samples_per_frame(audio_samples_per_frame) + , m_audio_frames_per_second(audio_frames_per_second) + , m_audio_samples_per_second(audio_samples_per_frame * audio_frames_per_second) + , m_microseconds_per_sample(1. / (m_audio_samples_per_second * 1000000.)) + , m_has_video(has_video) +{} + +void StreamHistoryTracker::set_window(std::chrono::seconds window){ + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + m_window = window; + clear_old(); +} +void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ + if (frames == 0){ + return; + } + WallClock now = current_time(); + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); +// cout << "on_samples() = " << m_audio.size() << endl; + m_audio.emplace_back(std::make_shared( + now, samples, frames * m_audio_samples_per_frame + )); + clear_old(); +} + + + + +void StreamHistoryTracker::on_frame(std::shared_ptr frame){ + // TODO: Find a more efficient way to buffer the frames. + // It takes almost 10GB of memory to store 30 seconds of QVideoFrames + // due to them caching uncompressed bitmaps. +// return; // TODO + + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); +// cout << "on_frame() = " << m_frames.size() << endl; + auto compressed_frame = compress_video_frame(frame->frame); + m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, compressed_frame}); + // m_frames.emplace_back(std::move(frame)); + clear_old(); +} + + + +void StreamHistoryTracker::clear_old(){ + // Must call under lock. + WallClock now = current_time(); + WallClock threshold = now - m_window; + +// WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); +// cout << "enter" << endl; + while (!m_audio.empty()){ +// cout << "audio.size() = " << m_audio.size() << endl; + AudioBlock& block = *m_audio.front(); + + WallClock end_block = block.timestamp; + end_block += std::chrono::microseconds( + static_cast((double)block.samples.size() * m_microseconds_per_sample) + ); + + if (end_block < threshold){ // todo: confirm if the audio deque clears properly + m_audio.pop_front(); + }else{ + break; + } + } +// cout << "exit" << endl; + + while (!m_compressed_frames.empty()){ + // if (m_frames.front()->timestamp < threshold){ + // m_frames.pop_front(); + // }else{ + // break; + // } + + if (m_compressed_frames.front().timestamp < threshold){ + m_compressed_frames.pop_front(); + }else{ + break; + } + } +} + + + + +bool StreamHistoryTracker::save(const std::string& filename) const{ + m_logger.log("Saving stream history...", COLOR_BLUE); + + std::deque frames; + { + // Fast copy the current state of the stream. + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + if (m_compressed_frames.empty()){ + return false; + } + frames = m_compressed_frames; + } + + // Now that the lock is released, we can take our time encoding it. + + // TODO + +#if 0 + WallClock start = WallClock::max(); + if (!frames.empty()){ + start = std::min(start, frames.front()->timestamp); + } + +#endif + + +// run_on_main_thread_and_wait([&]{ + + QVideoFrameFormat format(QSize(1920, 1080), QVideoFrameFormat::Format_ARGB8888); + QVideoFrameInput videoInput(format); + +// cout << "frames = " << frames.size() << endl; + + QMediaCaptureSession session; + QMediaRecorder recorder; + session.setVideoFrameInput(&videoInput); + session.setRecorder(&recorder); +#if 1 + recorder.setMediaFormat(QMediaFormat::MPEG4); +#else + QMediaFormat video_format; + video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); +// video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); + video_format.setFileFormat(QMediaFormat::MPEG4); + recorder.setMediaFormat(video_format); +#endif + recorder.setQuality(QMediaRecorder::NormalQuality); + + QFileInfo file(QString::fromStdString(filename)); + recorder.setOutputLocation( + QUrl::fromLocalFile(file.absoluteFilePath()) + ); + + VideoGenerator generator(&videoInput, frames); + + QObject::connect(&generator, &VideoGenerator::finished, &recorder, &QMediaRecorder::stop); + QObject::connect(&recorder, &QMediaRecorder::recorderStateChanged, [](QMediaRecorder::RecorderState state){ + if (state == QMediaRecorder::StoppedState) qApp->quit(); + }); + + recorder.record(); +#if 0 + WallClock last_change = current_time(); + bool success = true; + + while (!frames.empty()){ +#if 1 + while (true){ + if (frames.empty()){ +// video_input.sendVideoFrame(QVideoFrame()); +// session.setVideoFrameInput(nullptr); + break; + } + if (!video_input.sendVideoFrame((*frames.begin())->frame)){ +// cout << "Failed Video: " << frames.size() << endl; + break; + } + frames.pop_front(); + last_change = current_time(); +// cout << "Pushed Video: " << frames.size() << endl; + } +#endif + if (current_time() - last_change > std::chrono::seconds(10)){ + m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); + success = false; + break; + } + + QCoreApplication::processEvents(); + } +#endif + + recorder.stop(); + m_logger.log("Done saving stream history...", COLOR_BLUE); +// cout << recorder.duration() << endl; + + +// }); + return true; +} + + +#if 0 +bool StreamHistoryTracker::save(const std::string& filename) const{ + m_logger.log("Saving stream history...", COLOR_BLUE); + + std::deque> audio; + std::deque> frames; + { + // Fast copy the current state of the stream. + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + if (m_audio.empty() && m_frames.empty()){ + return false; + } + audio = m_audio; + frames = m_frames; + } + + // Now that the lock is released, we can take our time encoding it. + + // TODO + +#if 0 + WallClock start = WallClock::max(); + if (!audio.empty()){ + start = std::min(start, audio.front()->timestamp); + } + if (!frames.empty()){ + start = std::min(start, frames.front()->timestamp); + } + +#endif + + +// run_on_main_thread_and_wait([&]{ + + QAudioFormat audio_format; + audio_format.setChannelCount((int)m_audio_samples_per_frame); + audio_format.setChannelConfig(m_audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); + audio_format.setSampleRate((int)m_audio_frames_per_second); + audio_format.setSampleFormat(QAudioFormat::Float); + +// cout << "audio_format = " << audio_format.isValid() << endl; + + QAudioBufferInput audio_input; + QVideoFrameInput video_input; + +// cout << "audio = " << audio.size() << endl; +// cout << "frames = " << frames.size() << endl; + + QMediaCaptureSession session; + QMediaRecorder recorder; + session.setAudioBufferInput(&audio_input); + session.setVideoFrameInput(&video_input); + session.setRecorder(&recorder); +#if 1 + recorder.setMediaFormat(QMediaFormat::MPEG4); +#else + QMediaFormat video_format; + video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); +// video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); + video_format.setFileFormat(QMediaFormat::MPEG4); + recorder.setMediaFormat(video_format); +#endif + recorder.setQuality(QMediaRecorder::NormalQuality); + + QFileInfo file(QString::fromStdString(filename)); + recorder.setOutputLocation( + QUrl::fromLocalFile(file.absoluteFilePath()) + ); + + recorder.record(); + + WallClock last_change = current_time(); + QAudioBuffer audio_buffer; + bool success = true; + while (audio_buffer.isValid() || !frames.empty()){ +#if 1 + while (true){ + if (frames.empty()){ +// video_input.sendVideoFrame(QVideoFrame()); +// session.setVideoFrameInput(nullptr); + break; + } + if (!video_input.sendVideoFrame((*frames.begin())->frame)){ +// cout << "Failed Video: " << frames.size() << endl; + break; + } + frames.pop_front(); + last_change = current_time(); +// cout << "Pushed Video: " << frames.size() << endl; + } +#endif +#if 1 + while (true){ + if (!audio_buffer.isValid()){ + if (audio.empty()){ +// audio_input.sendAudioBuffer(QAudioBuffer()); +// session.setAudioBufferInput(nullptr); + break; + } +// cout << "constructing audio buffer: " << audio.size() << endl; + const std::vector& samples = audio.front()->samples; + QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); + audio_buffer = QAudioBuffer( + bytes, audio_format//, +// std::chrono::duration_cast(audio.front()->timestamp - start).count() + ); +// cout << "audio_buffer = " << audio_buffer.isValid() << endl; + audio.pop_front(); + } + if (!audio_buffer.isValid()){ + break; + } + if (!audio_input.sendAudioBuffer(audio_buffer)){ +// cout << "Failed Audio: " << audio.size() << endl; +// cout << audio_input.captureSession() << endl; + break; + } + audio_buffer = QAudioBuffer(); + last_change = current_time(); +// cout << "Pushed audio: " << audio.size() << endl; + } +#endif + + if (current_time() - last_change > std::chrono::seconds(10)){ + m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); + success = false; + break; + } + + QCoreApplication::processEvents(); + } + + recorder.stop(); + m_logger.log("Done saving stream history...", COLOR_BLUE); +// cout << recorder.duration() << endl; + + +// }); + return success; +} +#endif + + + + + + +} + +#include "StreamHistoryTracker_SaveFrames.moc" \ No newline at end of file diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index ee6ea1945f..dbb6e0ff85 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -10,25 +10,25 @@ #ifndef PokemonAutomation_StreamHistoryTracker_SaveFrames_H #define PokemonAutomation_StreamHistoryTracker_SaveFrames_H -#include +// #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "Common/Cpp/Logging/AbstractLogger.h" -#include "Common/Cpp/Concurrency/SpinLock.h" -#include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include "Common/Cpp/Logging/AbstractLogger.h" +// #include "Common/Cpp/Concurrency/SpinLock.h" +// #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" -#include -using std::cout; -using std::endl; +// #include +// using std::cout; +// using std::endl; namespace PokemonAutomation{ @@ -52,6 +52,8 @@ struct CompressedVideoFrame{ std::vector compressed_frame; }; +QVideoFrame decompress_video_frame(const std::vector &compressed_buffer); +std::vector compress_video_frame(const QVideoFrame& const_frame); class StreamHistoryTracker{ public: @@ -94,295 +96,6 @@ class StreamHistoryTracker{ -StreamHistoryTracker::StreamHistoryTracker( - Logger& logger, - std::chrono::seconds window, - size_t audio_samples_per_frame, - size_t audio_frames_per_second, - bool has_video -) - : m_logger(logger) - , m_window(window) - , m_audio_samples_per_frame(audio_samples_per_frame) - , m_audio_frames_per_second(audio_frames_per_second) - , m_audio_samples_per_second(audio_samples_per_frame * audio_frames_per_second) - , m_microseconds_per_sample(1. / (m_audio_samples_per_second * 1000000.)) - , m_has_video(has_video) -{} - -void StreamHistoryTracker::set_window(std::chrono::seconds window){ - WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - m_window = window; - clear_old(); -} -void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ - if (frames == 0){ - return; - } - WallClock now = current_time(); - WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); -// cout << "on_samples() = " << m_audio.size() << endl; - m_audio.emplace_back(std::make_shared( - now, samples, frames * m_audio_samples_per_frame - )); - clear_old(); -} - -std::vector compress_video_frame(const QVideoFrame& const_frame) { - // Create a local non-const copy (cheap, uses explicit sharing) - QVideoFrame frame = const_frame; - - // 1. Map the frame to CPU memory - if (!frame.map(QVideoFrame::ReadOnly)) { - return {}; - } - - // Ensure unmap() is called when this function exits (success or failure) - auto guard = qScopeGuard([&frame] { frame.unmap(); }); - - // 2. Convert to QImage (Qt 6.8+ handles internal conversions efficiently) - // For circular buffers, using a 3-channel RGB888 is common for OpenCV - QImage img = frame.toImage().convertToFormat(QImage::Format_RGB888); - - // 3. Wrap QImage memory into a cv::Mat (No-copy) - // Note: OpenCV expects BGR by default, but QImage is RGB. - // If color accuracy matters, use cv::cvtColor later or img.rgbSwapped(). - cv::Mat mat(img.height(), img.width(), CV_8UC3, - const_cast(img.bits()), img.bytesPerLine()); - - // 4. Compress using imencode - std::vector compressed_buffer; - std::vector params = {cv::IMWRITE_JPEG_QUALITY, 80}; // 0-100 - - // Convert RGB to BGR before encoding because imencode expects BGR - cv::Mat bgr_Mat; - cv::cvtColor(mat, bgr_Mat, cv::COLOR_RGB2BGR); - - cv::imencode(".jpg", bgr_Mat, compressed_buffer, params); - - return compressed_buffer; // Store this in your circular buffer -} - -QVideoFrame decompress_video_frame(const std::vector &compressed_buffer) { - if (compressed_buffer.empty()) return {}; - - // 1. Decompress JPEG buffer into a QImage - // fromData handles the JPEG header and decompression automatically - QImage img = QImage::fromData(compressed_buffer.data(), - static_cast(compressed_buffer.size()), - "JPG"); - - if (img.isNull()) return {}; - - // 2. Use the new Qt 6.8 constructor - // This wraps the QImage into a QVideoFrame efficiently. - // If the format is compatible (like RGB888), it minimizes copies. - return QVideoFrame(img); -} - -void StreamHistoryTracker::on_frame(std::shared_ptr frame){ - // TODO: Find a more efficient way to buffer the frames. - // It takes almost 10GB of memory to store 30 seconds of QVideoFrames - // due to them caching uncompressed bitmaps. -// return; // TODO - - WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); -// cout << "on_frame() = " << m_frames.size() << endl; - auto compressed_frame = compress_video_frame(frame->frame); - m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, compressed_frame}); - // m_frames.emplace_back(std::move(frame)); - clear_old(); -} - - - -void StreamHistoryTracker::clear_old(){ - // Must call under lock. - WallClock now = current_time(); - WallClock threshold = now - m_window; - -// WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); -// cout << "enter" << endl; - while (!m_audio.empty()){ -// cout << "audio.size() = " << m_audio.size() << endl; - AudioBlock& block = *m_audio.front(); - - WallClock end_block = block.timestamp; - end_block += std::chrono::microseconds( - static_cast((double)block.samples.size() * m_microseconds_per_sample) - ); - - if (end_block < threshold){ // todo: confirm if the audio deque clears properly - m_audio.pop_front(); - }else{ - break; - } - } -// cout << "exit" << endl; - - while (!m_compressed_frames.empty()){ - // if (m_frames.front()->timestamp < threshold){ - // m_frames.pop_front(); - // }else{ - // break; - // } - - if (m_compressed_frames.front().timestamp < threshold){ - m_compressed_frames.pop_front(); - }else{ - break; - } - } -} - - - - - -bool StreamHistoryTracker::save(const std::string& filename) const{ - m_logger.log("Saving stream history...", COLOR_BLUE); - - std::deque> audio; - std::deque> frames; - { - // Fast copy the current state of the stream. - WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_audio.empty() && m_frames.empty()){ - return false; - } - audio = m_audio; - frames = m_frames; - } - - // Now that the lock is released, we can take our time encoding it. - - // TODO - -#if 0 - WallClock start = WallClock::max(); - if (!audio.empty()){ - start = std::min(start, audio.front()->timestamp); - } - if (!frames.empty()){ - start = std::min(start, frames.front()->timestamp); - } - -#endif - - -// run_on_main_thread_and_wait([&]{ - - QAudioFormat audio_format; - audio_format.setChannelCount((int)m_audio_samples_per_frame); - audio_format.setChannelConfig(m_audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); - audio_format.setSampleRate((int)m_audio_frames_per_second); - audio_format.setSampleFormat(QAudioFormat::Float); - -// cout << "audio_format = " << audio_format.isValid() << endl; - - QAudioBufferInput audio_input; - QVideoFrameInput video_input; - -// cout << "audio = " << audio.size() << endl; -// cout << "frames = " << frames.size() << endl; - - QMediaCaptureSession session; - QMediaRecorder recorder; - session.setAudioBufferInput(&audio_input); - session.setVideoFrameInput(&video_input); - session.setRecorder(&recorder); -#if 1 - recorder.setMediaFormat(QMediaFormat::MPEG4); -#else - QMediaFormat video_format; - video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); -// video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); - video_format.setFileFormat(QMediaFormat::MPEG4); - recorder.setMediaFormat(video_format); -#endif - recorder.setQuality(QMediaRecorder::NormalQuality); - - QFileInfo file(QString::fromStdString(filename)); - recorder.setOutputLocation( - QUrl::fromLocalFile(file.absoluteFilePath()) - ); - - recorder.record(); - - WallClock last_change = current_time(); - QAudioBuffer audio_buffer; - bool success = true; - while (audio_buffer.isValid() || !frames.empty()){ -#if 1 - while (true){ - if (frames.empty()){ -// video_input.sendVideoFrame(QVideoFrame()); -// session.setVideoFrameInput(nullptr); - break; - } - if (!video_input.sendVideoFrame((*frames.begin())->frame)){ -// cout << "Failed Video: " << frames.size() << endl; - break; - } - frames.pop_front(); - last_change = current_time(); -// cout << "Pushed Video: " << frames.size() << endl; - } -#endif -#if 1 - while (true){ - if (!audio_buffer.isValid()){ - if (audio.empty()){ -// audio_input.sendAudioBuffer(QAudioBuffer()); -// session.setAudioBufferInput(nullptr); - break; - } -// cout << "constructing audio buffer: " << audio.size() << endl; - const std::vector& samples = audio.front()->samples; - QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); - audio_buffer = QAudioBuffer( - bytes, audio_format//, -// std::chrono::duration_cast(audio.front()->timestamp - start).count() - ); -// cout << "audio_buffer = " << audio_buffer.isValid() << endl; - audio.pop_front(); - } - if (!audio_buffer.isValid()){ - break; - } - if (!audio_input.sendAudioBuffer(audio_buffer)){ -// cout << "Failed Audio: " << audio.size() << endl; -// cout << audio_input.captureSession() << endl; - break; - } - audio_buffer = QAudioBuffer(); - last_change = current_time(); -// cout << "Pushed audio: " << audio.size() << endl; - } -#endif - - if (current_time() - last_change > std::chrono::seconds(10)){ - m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); - success = false; - break; - } - - QCoreApplication::processEvents(); - } - - recorder.stop(); - m_logger.log("Done saving stream history...", COLOR_BLUE); -// cout << recorder.duration() << endl; - - -// }); - return success; -} - - - - - } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index b54a42a58b..5ed9de82fd 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -471,6 +471,7 @@ file(GLOB LIBRARY_SOURCES Source/CommonFramework/Recording/StreamHistoryTracker_Null.h Source/CommonFramework/Recording/StreamHistoryTracker_ParallelStreams.h Source/CommonFramework/Recording/StreamHistoryTracker_RecordOnTheFly.h + Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h Source/CommonFramework/Recording/StreamRecorder.cpp Source/CommonFramework/Recording/StreamRecorder.h From 36913e8a0dfbe753b4c4384efe6943ae978f3852 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 27 Jan 2026 23:02:32 -0800 Subject: [PATCH 05/20] initial working implementation of saving frames to video. --- .../StreamHistoryTracker_SaveFrames.cpp | 488 ++++++++++-------- 1 file changed, 264 insertions(+), 224 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index 3d7328ccb2..fec06a8768 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -85,8 +85,9 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { } +#if 0 class VideoGenerator : public QObject { - Q_OBJECT + // Q_OBJECT public: VideoGenerator(QVideoFrameInput *input, std::deque frames) : m_input(input), m_frames(frames){ @@ -131,7 +132,7 @@ public slots: QVideoFrameInput *m_input; std::deque m_frames; }; - +#endif StreamHistoryTracker::StreamHistoryTracker( Logger& logger, @@ -226,8 +227,6 @@ void StreamHistoryTracker::clear_old(){ } - - bool StreamHistoryTracker::save(const std::string& filename) const{ m_logger.log("Saving stream history...", COLOR_BLUE); @@ -241,236 +240,277 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ frames = m_compressed_frames; } - // Now that the lock is released, we can take our time encoding it. + int width = 1920; + int height = 1080; - // TODO + // 1. Initialize VideoWriter (e.g., MP4 with 30 FPS) + cv::VideoWriter writer(filename, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), + 30.0, cv::Size(width, height), true); -#if 0 - WallClock start = WallClock::max(); - if (!frames.empty()){ - start = std::min(start, frames.front()->timestamp); + if (!writer.isOpened()) { + throw std::runtime_error("Could not open video file for writing."); } -#endif - - -// run_on_main_thread_and_wait([&]{ - - QVideoFrameFormat format(QSize(1920, 1080), QVideoFrameFormat::Format_ARGB8888); - QVideoFrameInput videoInput(format); - -// cout << "frames = " << frames.size() << endl; - - QMediaCaptureSession session; - QMediaRecorder recorder; - session.setVideoFrameInput(&videoInput); - session.setRecorder(&recorder); -#if 1 - recorder.setMediaFormat(QMediaFormat::MPEG4); -#else - QMediaFormat video_format; - video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); -// video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); - video_format.setFileFormat(QMediaFormat::MPEG4); - recorder.setMediaFormat(video_format); -#endif - recorder.setQuality(QMediaRecorder::NormalQuality); - - QFileInfo file(QString::fromStdString(filename)); - recorder.setOutputLocation( - QUrl::fromLocalFile(file.absoluteFilePath()) - ); - - VideoGenerator generator(&videoInput, frames); - - QObject::connect(&generator, &VideoGenerator::finished, &recorder, &QMediaRecorder::stop); - QObject::connect(&recorder, &QMediaRecorder::recorderStateChanged, [](QMediaRecorder::RecorderState state){ - if (state == QMediaRecorder::StoppedState) qApp->quit(); - }); - - recorder.record(); -#if 0 - WallClock last_change = current_time(); - bool success = true; - - while (!frames.empty()){ -#if 1 - while (true){ - if (frames.empty()){ -// video_input.sendVideoFrame(QVideoFrame()); -// session.setVideoFrameInput(nullptr); - break; - } - if (!video_input.sendVideoFrame((*frames.begin())->frame)){ -// cout << "Failed Video: " << frames.size() << endl; - break; - } - frames.pop_front(); - last_change = current_time(); -// cout << "Pushed Video: " << frames.size() << endl; - } -#endif - if (current_time() - last_change > std::chrono::seconds(10)){ - m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); - success = false; - break; - } - - QCoreApplication::processEvents(); + // 2. Loop through your memory pointers + for (CompressedVideoFrame frame : frames) { + QVideoFrame video_frame = decompress_video_frame(frame.compressed_frame); + QImage img = video_frame.toImage().convertToFormat(QImage::Format_BGR888); + + cv::Mat mat(height, width, CV_8UC3, (void*)img.bits(), img.bytesPerLine()); + + // 3. Write to video (Encoding happens here) + writer.write(mat); } -#endif + // Writer automatically releases when going out of scope - recorder.stop(); m_logger.log("Done saving stream history...", COLOR_BLUE); -// cout << recorder.duration() << endl; - - -// }); return true; } -#if 0 -bool StreamHistoryTracker::save(const std::string& filename) const{ - m_logger.log("Saving stream history...", COLOR_BLUE); - - std::deque> audio; - std::deque> frames; - { - // Fast copy the current state of the stream. - WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); - if (m_audio.empty() && m_frames.empty()){ - return false; - } - audio = m_audio; - frames = m_frames; - } - - // Now that the lock is released, we can take our time encoding it. - - // TODO - -#if 0 - WallClock start = WallClock::max(); - if (!audio.empty()){ - start = std::min(start, audio.front()->timestamp); - } - if (!frames.empty()){ - start = std::min(start, frames.front()->timestamp); - } - -#endif - - -// run_on_main_thread_and_wait([&]{ - - QAudioFormat audio_format; - audio_format.setChannelCount((int)m_audio_samples_per_frame); - audio_format.setChannelConfig(m_audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); - audio_format.setSampleRate((int)m_audio_frames_per_second); - audio_format.setSampleFormat(QAudioFormat::Float); - -// cout << "audio_format = " << audio_format.isValid() << endl; - - QAudioBufferInput audio_input; - QVideoFrameInput video_input; - -// cout << "audio = " << audio.size() << endl; -// cout << "frames = " << frames.size() << endl; - - QMediaCaptureSession session; - QMediaRecorder recorder; - session.setAudioBufferInput(&audio_input); - session.setVideoFrameInput(&video_input); - session.setRecorder(&recorder); -#if 1 - recorder.setMediaFormat(QMediaFormat::MPEG4); -#else - QMediaFormat video_format; - video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); -// video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); - video_format.setFileFormat(QMediaFormat::MPEG4); - recorder.setMediaFormat(video_format); -#endif - recorder.setQuality(QMediaRecorder::NormalQuality); - - QFileInfo file(QString::fromStdString(filename)); - recorder.setOutputLocation( - QUrl::fromLocalFile(file.absoluteFilePath()) - ); - - recorder.record(); - - WallClock last_change = current_time(); - QAudioBuffer audio_buffer; - bool success = true; - while (audio_buffer.isValid() || !frames.empty()){ -#if 1 - while (true){ - if (frames.empty()){ -// video_input.sendVideoFrame(QVideoFrame()); -// session.setVideoFrameInput(nullptr); - break; - } - if (!video_input.sendVideoFrame((*frames.begin())->frame)){ -// cout << "Failed Video: " << frames.size() << endl; - break; - } - frames.pop_front(); - last_change = current_time(); -// cout << "Pushed Video: " << frames.size() << endl; - } -#endif -#if 1 - while (true){ - if (!audio_buffer.isValid()){ - if (audio.empty()){ -// audio_input.sendAudioBuffer(QAudioBuffer()); -// session.setAudioBufferInput(nullptr); - break; - } -// cout << "constructing audio buffer: " << audio.size() << endl; - const std::vector& samples = audio.front()->samples; - QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); - audio_buffer = QAudioBuffer( - bytes, audio_format//, -// std::chrono::duration_cast(audio.front()->timestamp - start).count() - ); -// cout << "audio_buffer = " << audio_buffer.isValid() << endl; - audio.pop_front(); - } - if (!audio_buffer.isValid()){ - break; - } - if (!audio_input.sendAudioBuffer(audio_buffer)){ -// cout << "Failed Audio: " << audio.size() << endl; -// cout << audio_input.captureSession() << endl; - break; - } - audio_buffer = QAudioBuffer(); - last_change = current_time(); -// cout << "Pushed audio: " << audio.size() << endl; - } -#endif - - if (current_time() - last_change > std::chrono::seconds(10)){ - m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); - success = false; - break; - } - - QCoreApplication::processEvents(); - } - - recorder.stop(); - m_logger.log("Done saving stream history...", COLOR_BLUE); -// cout << recorder.duration() << endl; - - -// }); - return success; -} -#endif +// bool StreamHistoryTracker::save(const std::string& filename) const{ +// m_logger.log("Saving stream history...", COLOR_BLUE); + +// std::deque frames; +// { +// // Fast copy the current state of the stream. +// WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); +// if (m_compressed_frames.empty()){ +// return false; +// } +// frames = m_compressed_frames; +// } + +// // Now that the lock is released, we can take our time encoding it. + +// // TODO + +// #if 0 +// WallClock start = WallClock::max(); +// if (!frames.empty()){ +// start = std::min(start, frames.front()->timestamp); +// } + +// #endif + + +// // run_on_main_thread_and_wait([&]{ + +// QVideoFrameFormat format(QSize(1920, 1080), QVideoFrameFormat::Format_ARGB8888); +// QVideoFrameInput videoInput(format); + +// // cout << "frames = " << frames.size() << endl; + +// QMediaCaptureSession session; +// QMediaRecorder recorder; +// session.setVideoFrameInput(&videoInput); +// session.setRecorder(&recorder); +// #if 1 +// recorder.setMediaFormat(QMediaFormat::MPEG4); +// #else +// QMediaFormat video_format; +// video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); +// // video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); +// video_format.setFileFormat(QMediaFormat::MPEG4); +// recorder.setMediaFormat(video_format); +// #endif +// recorder.setQuality(QMediaRecorder::NormalQuality); + +// QFileInfo file(QString::fromStdString(filename)); +// recorder.setOutputLocation( +// QUrl::fromLocalFile(file.absoluteFilePath()) +// ); + +// VideoGenerator generator(&videoInput, frames); + +// QObject::connect(&generator, &VideoGenerator::finished, &recorder, &QMediaRecorder::stop); +// QObject::connect(&recorder, &QMediaRecorder::recorderStateChanged, [](QMediaRecorder::RecorderState state){ +// if (state == QMediaRecorder::StoppedState) qApp->quit(); +// }); + +// recorder.record(); +// #if 0 +// WallClock last_change = current_time(); +// bool success = true; + +// while (!frames.empty()){ +// #if 1 +// while (true){ +// if (frames.empty()){ +// // video_input.sendVideoFrame(QVideoFrame()); +// // session.setVideoFrameInput(nullptr); +// break; +// } +// if (!video_input.sendVideoFrame((*frames.begin())->frame)){ +// // cout << "Failed Video: " << frames.size() << endl; +// break; +// } +// frames.pop_front(); +// last_change = current_time(); +// // cout << "Pushed Video: " << frames.size() << endl; +// } +// #endif +// if (current_time() - last_change > std::chrono::seconds(10)){ +// m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); +// success = false; +// break; +// } + +// QCoreApplication::processEvents(); +// } +// #endif + +// recorder.stop(); +// m_logger.log("Done saving stream history...", COLOR_BLUE); +// // cout << recorder.duration() << endl; + + +// // }); +// return true; +// } + + +// #if 0 +// bool StreamHistoryTracker::save(const std::string& filename) const{ +// m_logger.log("Saving stream history...", COLOR_BLUE); + +// std::deque> audio; +// std::deque> frames; +// { +// // Fast copy the current state of the stream. +// WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); +// if (m_audio.empty() && m_frames.empty()){ +// return false; +// } +// audio = m_audio; +// frames = m_frames; +// } + +// // Now that the lock is released, we can take our time encoding it. + +// // TODO + +// #if 0 +// WallClock start = WallClock::max(); +// if (!audio.empty()){ +// start = std::min(start, audio.front()->timestamp); +// } +// if (!frames.empty()){ +// start = std::min(start, frames.front()->timestamp); +// } + +// #endif + + +// // run_on_main_thread_and_wait([&]{ + +// QAudioFormat audio_format; +// audio_format.setChannelCount((int)m_audio_samples_per_frame); +// audio_format.setChannelConfig(m_audio_samples_per_frame == 1 ? QAudioFormat::ChannelConfigMono : QAudioFormat::ChannelConfigStereo); +// audio_format.setSampleRate((int)m_audio_frames_per_second); +// audio_format.setSampleFormat(QAudioFormat::Float); + +// // cout << "audio_format = " << audio_format.isValid() << endl; + +// QAudioBufferInput audio_input; +// QVideoFrameInput video_input; + +// // cout << "audio = " << audio.size() << endl; +// // cout << "frames = " << frames.size() << endl; + +// QMediaCaptureSession session; +// QMediaRecorder recorder; +// session.setAudioBufferInput(&audio_input); +// session.setVideoFrameInput(&video_input); +// session.setRecorder(&recorder); +// #if 1 +// recorder.setMediaFormat(QMediaFormat::MPEG4); +// #else +// QMediaFormat video_format; +// video_format.setAudioCodec(QMediaFormat::AudioCodec::AAC); +// // video_format.setVideoCodec(QMediaFormat::VideoCodec::H264); +// video_format.setFileFormat(QMediaFormat::MPEG4); +// recorder.setMediaFormat(video_format); +// #endif +// recorder.setQuality(QMediaRecorder::NormalQuality); + +// QFileInfo file(QString::fromStdString(filename)); +// recorder.setOutputLocation( +// QUrl::fromLocalFile(file.absoluteFilePath()) +// ); + +// recorder.record(); + +// WallClock last_change = current_time(); +// QAudioBuffer audio_buffer; +// bool success = true; +// while (audio_buffer.isValid() || !frames.empty()){ +// #if 1 +// while (true){ +// if (frames.empty()){ +// // video_input.sendVideoFrame(QVideoFrame()); +// // session.setVideoFrameInput(nullptr); +// break; +// } +// if (!video_input.sendVideoFrame((*frames.begin())->frame)){ +// // cout << "Failed Video: " << frames.size() << endl; +// break; +// } +// frames.pop_front(); +// last_change = current_time(); +// // cout << "Pushed Video: " << frames.size() << endl; +// } +// #endif +// #if 1 +// while (true){ +// if (!audio_buffer.isValid()){ +// if (audio.empty()){ +// // audio_input.sendAudioBuffer(QAudioBuffer()); +// // session.setAudioBufferInput(nullptr); +// break; +// } +// // cout << "constructing audio buffer: " << audio.size() << endl; +// const std::vector& samples = audio.front()->samples; +// QByteArray bytes((const char*)samples.data(), samples.size() * sizeof(float)); +// audio_buffer = QAudioBuffer( +// bytes, audio_format//, +// // std::chrono::duration_cast(audio.front()->timestamp - start).count() +// ); +// // cout << "audio_buffer = " << audio_buffer.isValid() << endl; +// audio.pop_front(); +// } +// if (!audio_buffer.isValid()){ +// break; +// } +// if (!audio_input.sendAudioBuffer(audio_buffer)){ +// // cout << "Failed Audio: " << audio.size() << endl; +// // cout << audio_input.captureSession() << endl; +// break; +// } +// audio_buffer = QAudioBuffer(); +// last_change = current_time(); +// // cout << "Pushed audio: " << audio.size() << endl; +// } +// #endif + +// if (current_time() - last_change > std::chrono::seconds(10)){ +// m_logger.log("Failed to record stream history: No progress made after 10 seconds.", COLOR_RED); +// success = false; +// break; +// } + +// QCoreApplication::processEvents(); +// } + +// recorder.stop(); +// m_logger.log("Done saving stream history...", COLOR_BLUE); +// // cout << recorder.duration() << endl; + + +// // }); +// return success; +// } +// #endif @@ -479,4 +519,4 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ } -#include "StreamHistoryTracker_SaveFrames.moc" \ No newline at end of file +// #include "StreamHistoryTracker_SaveFrames.moc" \ No newline at end of file From 167711cb10310028530beb459489b079bcc462e9 Mon Sep 17 00:00:00 2001 From: jw098 Date: Wed, 28 Jan 2026 22:18:17 -0800 Subject: [PATCH 06/20] reduce video file size --- .../StreamHistoryTracker_SaveFrames.cpp | 34 +++++++++++++------ .../StreamHistoryTracker_SaveFrames.h | 2 +- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index fec06a8768..aec12f82d3 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -32,7 +32,7 @@ namespace PokemonAutomation{ -QVideoFrame decompress_video_frame(const std::vector &compressed_buffer) { +QImage decompress_video_frame(const std::vector &compressed_buffer) { if (compressed_buffer.empty()) return {}; // 1. Decompress JPEG buffer into a QImage @@ -43,10 +43,7 @@ QVideoFrame decompress_video_frame(const std::vector &compressed_buffer) if (img.isNull()) return {}; - // 2. Use the new Qt 6.8 constructor - // This wraps the QImage into a QVideoFrame efficiently. - // If the format is compatible (like RGB888), it minimizes copies. - return QVideoFrame(img); + return img.convertToFormat(QImage::Format_BGR888); } std::vector compress_video_frame(const QVideoFrame& const_frame) { @@ -65,6 +62,12 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { // For circular buffers, using a 3-channel RGB888 is common for OpenCV QImage img = frame.toImage().convertToFormat(QImage::Format_RGB888); + // downscale to 720p for smaller file size + int target_width = 1280; + int target_height = img.height() * target_width / img.width(); + img = img.scaled(target_width, target_height, Qt::KeepAspectRatio, Qt::SmoothTransformation); + + // 3. Wrap QImage memory into a cv::Mat (No-copy) // Note: OpenCV expects BGR by default, but QImage is RGB. // If color accuracy matters, use cv::cvtColor later or img.rgbSwapped(). @@ -73,7 +76,7 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { // 4. Compress using imencode std::vector compressed_buffer; - std::vector params = {cv::IMWRITE_JPEG_QUALITY, 80}; // 0-100 + std::vector params = {cv::IMWRITE_JPEG_QUALITY, 50}; // 0-100 // Convert RGB to BGR before encoding because imencode expects BGR cv::Mat bgr_Mat; @@ -180,7 +183,7 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); // cout << "on_frame() = " << m_frames.size() << endl; auto compressed_frame = compress_video_frame(frame->frame); - m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, compressed_frame}); + m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, std::move(compressed_frame)}); // m_frames.emplace_back(std::move(frame)); clear_old(); } @@ -240,8 +243,16 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ frames = m_compressed_frames; } - int width = 1920; - int height = 1080; + if (frames.empty()) return false; + + // Use first frame to get size + // QVideoFrame first_video_frame = decompress_video_frame(frames.front().compressed_frame); + // QImage first_img = first_video_frame.toImage().convertToFormat(QImage::Format_BGR888); + QImage first_img = decompress_video_frame(frames.front().compressed_frame); + int width = first_img.width(); + int height = first_img.height(); + + cout << width << endl; // 1. Initialize VideoWriter (e.g., MP4 with 30 FPS) cv::VideoWriter writer(filename, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), @@ -253,8 +264,9 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ // 2. Loop through your memory pointers for (CompressedVideoFrame frame : frames) { - QVideoFrame video_frame = decompress_video_frame(frame.compressed_frame); - QImage img = video_frame.toImage().convertToFormat(QImage::Format_BGR888); + // QVideoFrame video_frame = decompress_video_frame(frame.compressed_frame); + // QImage img = video_frame.toImage().convertToFormat(QImage::Format_BGR888); + QImage img = decompress_video_frame(frame.compressed_frame); cv::Mat mat(height, width, CV_8UC3, (void*)img.bits(), img.bytesPerLine()); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index dbb6e0ff85..9771075790 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -52,7 +52,7 @@ struct CompressedVideoFrame{ std::vector compressed_frame; }; -QVideoFrame decompress_video_frame(const std::vector &compressed_buffer); +QImage decompress_video_frame(const std::vector &compressed_buffer); std::vector compress_video_frame(const QVideoFrame& const_frame); class StreamHistoryTracker{ From 01a5e018465aba7ffc2942a2274dc1e916c41b0d Mon Sep 17 00:00:00 2001 From: jw098 Date: Wed, 28 Jan 2026 22:37:34 -0800 Subject: [PATCH 07/20] reduce fps --- .../Recording/StreamHistoryTracker_SaveFrames.cpp | 11 ++++++++++- .../Recording/StreamHistoryTracker_SaveFrames.h | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index aec12f82d3..f4fa5a616d 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -151,6 +151,7 @@ StreamHistoryTracker::StreamHistoryTracker( , m_audio_samples_per_second(audio_samples_per_frame * audio_frames_per_second) , m_microseconds_per_sample(1. / (m_audio_samples_per_second * 1000000.)) , m_has_video(has_video) + , m_target_fps(15) {} void StreamHistoryTracker::set_window(std::chrono::seconds window){ @@ -182,6 +183,14 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); // cout << "on_frame() = " << m_frames.size() << endl; + m_frame_counter++; + size_t source_fps = 30; + size_t keep_nth_frame = source_fps / m_target_fps; + // Only keep every nth frame + if (m_frame_counter % keep_nth_frame != 0){ + return; + } + auto compressed_frame = compress_video_frame(frame->frame); m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, std::move(compressed_frame)}); // m_frames.emplace_back(std::move(frame)); @@ -256,7 +265,7 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ // 1. Initialize VideoWriter (e.g., MP4 with 30 FPS) cv::VideoWriter writer(filename, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), - 30.0, cv::Size(width, height), true); + m_target_fps, cv::Size(width, height), true); if (!writer.isOpened()) { throw std::runtime_error("Could not open video file for writing."); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 9771075790..956f3dcddd 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -85,12 +85,15 @@ class StreamHistoryTracker{ const size_t m_audio_samples_per_second; const double m_microseconds_per_sample; const bool m_has_video; + size_t m_target_fps; // We use shared_ptr here so it's fast to snapshot when we need to copy // everything asynchronously. std::deque> m_audio; std::deque> m_frames; std::deque m_compressed_frames; + size_t m_frame_counter = 0; + }; From 443c20cc5224c37b846d9aaa20480210cd42bc60 Mon Sep 17 00:00:00 2001 From: jw098 Date: Wed, 28 Jan 2026 23:39:20 -0800 Subject: [PATCH 08/20] adjust implementation of variable FPS --- .../StreamHistoryTracker_SaveFrames.cpp | 42 +++++++++++-------- .../StreamHistoryTracker_SaveFrames.h | 3 +- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index f4fa5a616d..086a8ab45d 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -152,6 +152,7 @@ StreamHistoryTracker::StreamHistoryTracker( , m_microseconds_per_sample(1. / (m_audio_samples_per_second * 1000000.)) , m_has_video(has_video) , m_target_fps(15) + , m_frame_interval(1000000 / m_target_fps) {} void StreamHistoryTracker::set_window(std::chrono::seconds window){ @@ -160,6 +161,7 @@ void StreamHistoryTracker::set_window(std::chrono::seconds window){ clear_old(); } void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ + #if 0 if (frames == 0){ return; } @@ -170,25 +172,31 @@ void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ now, samples, frames * m_audio_samples_per_frame )); clear_old(); + #endif } void StreamHistoryTracker::on_frame(std::shared_ptr frame){ - // TODO: Find a more efficient way to buffer the frames. - // It takes almost 10GB of memory to store 30 seconds of QVideoFrames - // due to them caching uncompressed bitmaps. -// return; // TODO WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); // cout << "on_frame() = " << m_frames.size() << endl; - m_frame_counter++; - size_t source_fps = 30; - size_t keep_nth_frame = source_fps / m_target_fps; - // Only keep every nth frame - if (m_frame_counter % keep_nth_frame != 0){ - return; + + // Initialize on first frame + if (m_next_frame_time == WallClock{}){ + m_next_frame_time = frame->timestamp; + } + + // don't save every frame. only save frames as per m_target_fps + // Only save when we've crossed the next sampling boundary + if (frame->timestamp < m_next_frame_time){ + return; // skip + } + + // Advance by fixed intervals (NOT by arrival time) + while (m_next_frame_time <= frame->timestamp){ + m_next_frame_time += std::chrono::microseconds(m_frame_interval); } auto compressed_frame = compress_video_frame(frame->frame); @@ -201,9 +209,10 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ void StreamHistoryTracker::clear_old(){ // Must call under lock. - WallClock now = current_time(); - WallClock threshold = now - m_window; + WallClock latest_frame = m_compressed_frames.back().timestamp; + WallClock threshold = latest_frame - m_window; + #if 0 // WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); // cout << "enter" << endl; while (!m_audio.empty()){ @@ -221,15 +230,10 @@ void StreamHistoryTracker::clear_old(){ break; } } + #endif // cout << "exit" << endl; while (!m_compressed_frames.empty()){ - // if (m_frames.front()->timestamp < threshold){ - // m_frames.pop_front(); - // }else{ - // break; - // } - if (m_compressed_frames.front().timestamp < threshold){ m_compressed_frames.pop_front(); }else{ @@ -252,6 +256,8 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ frames = m_compressed_frames; } + cout << "frames.size(): " << frames.size() << endl; + if (frames.empty()) return false; // Use first frame to get size diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 956f3dcddd..4620bdfc5e 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -86,13 +86,14 @@ class StreamHistoryTracker{ const double m_microseconds_per_sample; const bool m_has_video; size_t m_target_fps; + std::chrono::microseconds m_frame_interval; // We use shared_ptr here so it's fast to snapshot when we need to copy // everything asynchronously. std::deque> m_audio; std::deque> m_frames; std::deque m_compressed_frames; - size_t m_frame_counter = 0; + WallClock m_next_frame_time; }; From abdf5bfca065e57a5abcb5e10c5180372c551ee3 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 29 Jan 2026 20:58:36 -0800 Subject: [PATCH 09/20] add options to change FPS, JPEG quality. --- .../Recording/StreamHistoryOption.cpp | 31 ++++++++++-- .../Recording/StreamHistoryOption.h | 11 ++++ .../StreamHistoryTracker_SaveFrames.cpp | 50 +++++++++++++++++-- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp index 448a3e7566..9dc485387f 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp @@ -80,13 +80,38 @@ StreamHistoryOption::StreamHistoryOption() LockMode::UNLOCK_WHILE_RUNNING, 5000 ) + , VIDEO_FPS( + "Video Frames per second:
" + "Lower = choppier video, smaller file size.
" + "Higher = smoother video, larger file size.", + { + // {VideoFPS::MATCH_INPUT, "match", "Match Input FPS"}, + {VideoFPS::FPS_30, "fps-30", "30 FPS"}, + {VideoFPS::FPS_15, "fps-15", "15 FPS"}, + {VideoFPS::FPS_10, "fps-10", "10 FPS"}, + {VideoFPS::FPS_05, "fps-05", "5 FPS"}, + {VideoFPS::FPS_01, "fps-01", "1 FPS"}, + }, + LockMode::UNLOCK_WHILE_RUNNING, + VideoFPS::FPS_15 + ) + , JPEG_QUALITY( + "JPEG Quality (video frames are compressed into JPEGs to save space in RAM):
" + "Lower = lower quality, smaller file size.
" + "Higher = high quality, larger file size.", + LockMode::UNLOCK_WHILE_RUNNING, + 80, + 0, 100 + ) { PA_ADD_STATIC(DESCRIPTION); PA_ADD_OPTION(HISTORY_SECONDS); PA_ADD_OPTION(RESOLUTION); - PA_ADD_OPTION(ENCODING_MODE); - PA_ADD_OPTION(VIDEO_QUALITY); - PA_ADD_OPTION(VIDEO_BITRATE); + PA_ADD_OPTION(VIDEO_FPS); + PA_ADD_OPTION(JPEG_QUALITY); + // PA_ADD_OPTION(ENCODING_MODE); + // PA_ADD_OPTION(VIDEO_QUALITY); + // PA_ADD_OPTION(VIDEO_BITRATE); StreamHistoryOption::on_config_value_changed(this); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.h index 11781c302f..3ef7ea0b6b 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.h @@ -49,6 +49,17 @@ class StreamHistoryOption : public GroupOption, public ConfigOption::Listener{ }; EnumDropdownOption VIDEO_QUALITY; SimpleIntegerOption VIDEO_BITRATE; + + enum class VideoFPS{ + // MATCH_INPUT, + FPS_30, + FPS_15, + FPS_10, + FPS_05, + FPS_01, + }; + EnumDropdownOption VIDEO_FPS; + SimpleIntegerOption JPEG_QUALITY; }; diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index 086a8ab45d..c42f94bea6 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -18,9 +18,12 @@ #include #include #include +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Concurrency/SpinLock.h" #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" +#include "CommonFramework/GlobalSettingsPanel.h" +#include "CommonFramework/Recording/StreamHistoryOption.h" #include "StreamHistoryTracker_SaveFrames.h" #include @@ -62,8 +65,23 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { // For circular buffers, using a 3-channel RGB888 is common for OpenCV QImage img = frame.toImage().convertToFormat(QImage::Format_RGB888); + int target_width; + const StreamHistoryOption& settings = GlobalSettings::instance().STREAM_HISTORY; + switch (settings.RESOLUTION){ + case StreamHistoryOption::Resolution::MATCH_INPUT: + target_width = img.width(); + break; + case StreamHistoryOption::Resolution::FORCE_720p: + target_width = 1280; + break; + case StreamHistoryOption::Resolution::FORCE_1080p: + target_width = 1920; + break; + default: + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Resolution: Unknown enum."); + } + // downscale to 720p for smaller file size - int target_width = 1280; int target_height = img.height() * target_width / img.width(); img = img.scaled(target_width, target_height, Qt::KeepAspectRatio, Qt::SmoothTransformation); @@ -76,7 +94,7 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { // 4. Compress using imencode std::vector compressed_buffer; - std::vector params = {cv::IMWRITE_JPEG_QUALITY, 50}; // 0-100 + std::vector params = {cv::IMWRITE_JPEG_QUALITY, settings.JPEG_QUALITY}; // 0-100 // Convert RGB to BGR before encoding because imencode expects BGR cv::Mat bgr_Mat; @@ -87,6 +105,32 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { return compressed_buffer; // Store this in your circular buffer } +size_t get_target_fps(){ + const StreamHistoryOption& settings = GlobalSettings::instance().STREAM_HISTORY; + size_t target_fps; + switch (settings.VIDEO_FPS){ + case StreamHistoryOption::VideoFPS::FPS_30: + target_fps = 30; + break; + case StreamHistoryOption::VideoFPS::FPS_15: + target_fps = 15; + break; + case StreamHistoryOption::VideoFPS::FPS_10: + target_fps = 10; + break; + case StreamHistoryOption::VideoFPS::FPS_05: + target_fps = 5; + break; + case StreamHistoryOption::VideoFPS::FPS_01: + target_fps = 1; + break; + default: + throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "VideoFPS: Unknown enum."); + } + + return target_fps; +} + #if 0 class VideoGenerator : public QObject { @@ -151,7 +195,7 @@ StreamHistoryTracker::StreamHistoryTracker( , m_audio_samples_per_second(audio_samples_per_frame * audio_frames_per_second) , m_microseconds_per_sample(1. / (m_audio_samples_per_second * 1000000.)) , m_has_video(has_video) - , m_target_fps(15) + , m_target_fps(get_target_fps()) , m_frame_interval(1000000 / m_target_fps) {} From ca302c9b59147fd63ef513683207301cc197b326 Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 29 Jan 2026 21:01:06 -0800 Subject: [PATCH 10/20] re-enable StreamHistoryTracker_Null --- .../Source/CommonFramework/Recording/StreamHistorySession.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp index 95faa7b6fe..6e5c071d2b 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistorySession.cpp @@ -17,7 +17,7 @@ //#include "StreamHistoryTracker_RecordOnTheFly.h" // #include "StreamHistoryTracker_ParallelStreams.h" #else -// #include "StreamHistoryTracker_Null.h" +#include "StreamHistoryTracker_Null.h" #endif From d28eb9cee6e204815eec4c16b501cff2a5864aac Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 29 Jan 2026 21:02:42 -0800 Subject: [PATCH 11/20] remove old code. --- .../StreamHistoryTracker_SaveFrames.h | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 4620bdfc5e..7dda02331f 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -10,25 +10,7 @@ #ifndef PokemonAutomation_StreamHistoryTracker_SaveFrames_H #define PokemonAutomation_StreamHistoryTracker_SaveFrames_H -// #include #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include "Common/Cpp/Logging/AbstractLogger.h" -// #include "Common/Cpp/Concurrency/SpinLock.h" -// #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" - - -// #include -// using std::cout; -// using std::endl; namespace PokemonAutomation{ From 2815e7f31f833b20a42a5fe2277abbc3564e27dd Mon Sep 17 00:00:00 2001 From: jw098 Date: Thu, 29 Jan 2026 22:29:35 -0800 Subject: [PATCH 12/20] update StreamHistoryOption text --- .../Recording/StreamHistoryOption.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp index 9dc485387f..7975ff5e02 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryOption.cpp @@ -17,15 +17,15 @@ StreamHistoryOption::StreamHistoryOption() true ) , DESCRIPTION( - "Keep a record of the recent video+audio streams. This will allow video capture " + "Keep a record of the recent video streams. This will allow video capture " "for unexpected events.

" - "Warning: This feature has a known memory leak. It will leak ~3GB per day per " - "video stream. You have been warned!" - "

" + // "Warning: This feature has a known memory leak. It will leak ~3GB per day per " + // "video stream. You have been warned!" + // "

" "Warning: This feature is computationally expensive and " "will require a more powerful computer to run (especially for multi-Switch programs).
" - "Furthermore, the current implementation is inefficient as it will write a lot " - "of data to disk. This feature is still a work-in-progress." + // "Furthermore, the current implementation is inefficient as it will write a lot " + // "of data to disk. This feature is still a work-in-progress." "" ) , HISTORY_SECONDS( @@ -96,9 +96,10 @@ StreamHistoryOption::StreamHistoryOption() VideoFPS::FPS_15 ) , JPEG_QUALITY( - "JPEG Quality (video frames are compressed into JPEGs to save space in RAM):
" - "Lower = lower quality, smaller file size.
" - "Higher = high quality, larger file size.", + "JPEG Quality:
" + "Video frames are compressed into JPEGs to save space in RAM.
" + "Lower = lower quality, lower RAM usage.
" + "Higher = high quality, higher RAM usage.", LockMode::UNLOCK_WHILE_RUNNING, 80, 0, 100 From 6f9f0664cb79c36a5a67a071cb9df3d8f890f5a2 Mon Sep 17 00:00:00 2001 From: jw098 Date: Mon, 2 Feb 2026 22:53:31 -0800 Subject: [PATCH 13/20] move compression to separate thread. --- .../StreamHistoryTracker_SaveFrames.cpp | 108 +++++++++++++++--- .../StreamHistoryTracker_SaveFrames.h | 14 +++ 2 files changed, 103 insertions(+), 19 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index c42f94bea6..d2c74106f8 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -21,6 +21,7 @@ #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Concurrency/SpinLock.h" +#include "Common/Cpp/Concurrency/Mutex.h" #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Recording/StreamHistoryOption.h" @@ -35,6 +36,15 @@ namespace PokemonAutomation{ +void simulate_cpu_load(int milliseconds) { + auto start = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() - start < std::chrono::milliseconds(milliseconds)) { + // Waste cycles with dummy math to prevent compiler optimization + double d = 1.0; + d = std::sqrt(d * 1.1); + } +} + QImage decompress_video_frame(const std::vector &compressed_buffer) { if (compressed_buffer.empty()) return {}; @@ -50,6 +60,9 @@ QImage decompress_video_frame(const std::vector &compressed_buffer) { } std::vector compress_video_frame(const QVideoFrame& const_frame) { + simulate_cpu_load(100); // for testing, what happens when the CPU is overwhelmed, and needs to drop frames. + + // Create a local non-const copy (cheap, uses explicit sharing) QVideoFrame frame = const_frame; @@ -181,6 +194,14 @@ public slots: }; #endif +StreamHistoryTracker::~StreamHistoryTracker() { + m_stopping = true; + m_cv.notify_all(); + if (m_worker.joinable()) { + m_worker.join(); + } +} + StreamHistoryTracker::StreamHistoryTracker( Logger& logger, std::chrono::seconds window, @@ -197,7 +218,9 @@ StreamHistoryTracker::StreamHistoryTracker( , m_has_video(has_video) , m_target_fps(get_target_fps()) , m_frame_interval(1000000 / m_target_fps) -{} +{ + m_worker = std::thread(&StreamHistoryTracker::worker_loop, this); +} void StreamHistoryTracker::set_window(std::chrono::seconds window){ WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); @@ -223,30 +246,46 @@ void StreamHistoryTracker::on_samples(const float* samples, size_t frames){ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ + { + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + // cout << "on_frame() = " << m_frames.size() << endl; - WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); -// cout << "on_frame() = " << m_frames.size() << endl; + // Initialize on first frame + if (m_next_frame_time == WallClock{}){ + m_next_frame_time = frame->timestamp; + } - // Initialize on first frame - if (m_next_frame_time == WallClock{}){ - m_next_frame_time = frame->timestamp; - } + // don't save every frame. only save frames as per m_target_fps + // Only save when we've crossed the next sampling boundary + if (frame->timestamp < m_next_frame_time){ + return; // skip + } + + // Advance by fixed intervals (NOT by arrival time) + while (m_next_frame_time <= frame->timestamp){ + m_next_frame_time += std::chrono::microseconds(m_frame_interval); + } + } // Release SpinLock before hitting the queue mutex - // don't save every frame. only save frames as per m_target_fps - // Only save when we've crossed the next sampling boundary - if (frame->timestamp < m_next_frame_time){ - return; // skip - } - // Advance by fixed intervals (NOT by arrival time) - while (m_next_frame_time <= frame->timestamp){ - m_next_frame_time += std::chrono::microseconds(m_frame_interval); + // auto compressed_frame = compress_video_frame(frame->frame); + // m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, std::move(compressed_frame)}); + // // m_frames.emplace_back(std::move(frame)); + // clear_old(); + + { + std::lock_guard lock(m_queue_lock); + + // Drop oldest if we are falling behind + if (m_pending_frames.size() >= MAX_PENDING_FRAMES) { + m_pending_frames.pop_front(); + m_logger.log("Worker thread lagging: Frame dropped.", COLOR_RED); + } + m_pending_frames.emplace_back(std::move(frame)); + } + m_cv.notify_one(); - auto compressed_frame = compress_video_frame(frame->frame); - m_compressed_frames.emplace_back(CompressedVideoFrame{frame->timestamp, std::move(compressed_frame)}); - // m_frames.emplace_back(std::move(frame)); - clear_old(); } @@ -339,6 +378,37 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ } +void StreamHistoryTracker::worker_loop() { + while (!m_stopping) { + std::shared_ptr frame; + + // 1. Wait for a frame to process + { + std::unique_lock lock(m_queue_lock); + m_cv.wait(lock, [this] { return !m_pending_frames.empty() || m_stopping; }); + + if (m_stopping && m_pending_frames.empty()) return; + + frame = std::move(m_pending_frames.front()); + m_pending_frames.pop_front(); + } + + // 2. Perform the expensive compression (Outside the lock) + auto compressed_data = compress_video_frame(frame->frame); + + // 3. Move the result into the main storage + { + WriteSpinLock lg(m_lock, PA_CURRENT_FUNCTION); + m_compressed_frames.emplace_back(CompressedVideoFrame{ + frame->timestamp, + std::move(compressed_data) + }); + clear_old(); // Cleanup happens here + } + } +} + + // bool StreamHistoryTracker::save(const std::string& filename) const{ // m_logger.log("Saving stream history...", COLOR_BLUE); diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 7dda02331f..697ba4652d 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -39,6 +39,7 @@ std::vector compress_video_frame(const QVideoFrame& const_frame); class StreamHistoryTracker{ public: + ~StreamHistoryTracker(); StreamHistoryTracker( Logger& logger, std::chrono::seconds window, @@ -56,8 +57,10 @@ class StreamHistoryTracker{ private: void clear_old(); + void worker_loop(); // The function that runs in the thread private: + static constexpr size_t MAX_PENDING_FRAMES = 10; Logger& m_logger; mutable SpinLock m_lock; std::chrono::seconds m_window; @@ -76,6 +79,17 @@ class StreamHistoryTracker{ std::deque> m_frames; std::deque m_compressed_frames; WallClock m_next_frame_time; + + std::thread m_worker; + std::atomic m_stopping{false}; + + // Queue for the worker thread + std::mutex m_queue_lock; + std::condition_variable m_cv; + std::deque> m_pending_frames; + + + }; From 20bb3104635ed13011f8090baab255cd72b0823b Mon Sep 17 00:00:00 2001 From: jw098 Date: Mon, 2 Feb 2026 23:57:04 -0800 Subject: [PATCH 14/20] Insert dummy frames if there is a gap due to dropping frames, to preserve frame rate --- .../StreamHistoryTracker_SaveFrames.cpp | 37 ++++++++++++------- .../StreamHistoryTracker_SaveFrames.h | 6 ++- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index d2c74106f8..ebb20d46df 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -21,7 +21,6 @@ #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Concurrency/SpinLock.h" -#include "Common/Cpp/Concurrency/Mutex.h" #include "CommonFramework/VideoPipeline/Backends/VideoFrameQt.h" #include "CommonFramework/GlobalSettingsPanel.h" #include "CommonFramework/Recording/StreamHistoryOption.h" @@ -60,7 +59,7 @@ QImage decompress_video_frame(const std::vector &compressed_buffer) { } std::vector compress_video_frame(const QVideoFrame& const_frame) { - simulate_cpu_load(100); // for testing, what happens when the CPU is overwhelmed, and needs to drop frames. + // simulate_cpu_load(100); // for testing, to see what happens when the CPU is overwhelmed, and needs to drop frames. // Create a local non-const copy (cheap, uses explicit sharing) @@ -94,14 +93,14 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Resolution: Unknown enum."); } - // downscale to 720p for smaller file size + // scale to target resolution int target_height = img.height() * target_width / img.width(); img = img.scaled(target_width, target_height, Qt::KeepAspectRatio, Qt::SmoothTransformation); // 3. Wrap QImage memory into a cv::Mat (No-copy) // Note: OpenCV expects BGR by default, but QImage is RGB. - // If color accuracy matters, use cv::cvtColor later or img.rgbSwapped(). + // we use cv::cvtColor later to fix this cv::Mat mat(img.height(), img.width(), CV_8UC3, const_cast(img.bits()), img.bytesPerLine()); @@ -115,7 +114,7 @@ std::vector compress_video_frame(const QVideoFrame& const_frame) { cv::imencode(".jpg", bgr_Mat, compressed_buffer, params); - return compressed_buffer; // Store this in your circular buffer + return compressed_buffer; // Store this in the circular buffer } size_t get_target_fps(){ @@ -261,7 +260,7 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ return; // skip } - // Advance by fixed intervals (NOT by arrival time) + // Advance by fixed intervals while (m_next_frame_time <= frame->timestamp){ m_next_frame_time += std::chrono::microseconds(m_frame_interval); } @@ -344,8 +343,6 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ if (frames.empty()) return false; // Use first frame to get size - // QVideoFrame first_video_frame = decompress_video_frame(frames.front().compressed_frame); - // QImage first_img = first_video_frame.toImage().convertToFormat(QImage::Format_BGR888); QImage first_img = decompress_video_frame(frames.front().compressed_frame); int width = first_img.width(); int height = first_img.height(); @@ -360,15 +357,29 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ throw std::runtime_error("Could not open video file for writing."); } - // 2. Loop through your memory pointers + std::vector last_good_buffer = frames[0].compressed_frame; + WallClock current_timeline = frames[0].timestamp; + + // 2. Loop through frames for (CompressedVideoFrame frame : frames) { - // QVideoFrame video_frame = decompress_video_frame(frame.compressed_frame); - // QImage img = video_frame.toImage().convertToFormat(QImage::Format_BGR888); + // Insert dummy frames if there is a gap due to dropping frames. + // Because VideoWriter can only handle a fixed frame rate. + while (current_timeline + m_frame_interval < frame.timestamp) { + // Decompress last known good frame and write again + QImage img = decompress_video_frame(last_good_buffer); + cv::Mat mat(height, width, CV_8UC3, (void*)img.bits(), img.bytesPerLine()); + writer.write(mat); + + current_timeline += m_frame_interval; + } + QImage img = decompress_video_frame(frame.compressed_frame); cv::Mat mat(height, width, CV_8UC3, (void*)img.bits(), img.bytesPerLine()); - - // 3. Write to video (Encoding happens here) + last_good_buffer = frame.compressed_frame; + current_timeline = frame.timestamp; + + // 3. Write to video writer.write(mat); } // Writer automatically releases when going out of scope diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 697ba4652d..2f5fe7368b 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -11,6 +11,8 @@ #define PokemonAutomation_StreamHistoryTracker_SaveFrames_H #include +#include "Common/Cpp/Concurrency/Mutex.h" +#include "Common/Cpp/Concurrency/ConditionVariable.h" namespace PokemonAutomation{ @@ -84,8 +86,8 @@ class StreamHistoryTracker{ std::atomic m_stopping{false}; // Queue for the worker thread - std::mutex m_queue_lock; - std::condition_variable m_cv; + Mutex m_queue_lock; + ConditionVariable m_cv; std::deque> m_pending_frames; From fa276f7f646dfe074533ae7cdc2ed49aede3d9c3 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 3 Feb 2026 12:48:56 -0800 Subject: [PATCH 15/20] add comments --- .../Recording/StreamHistoryTracker_SaveFrames.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index ebb20d46df..cf9de41552 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -257,10 +257,17 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ // don't save every frame. only save frames as per m_target_fps // Only save when we've crossed the next sampling boundary if (frame->timestamp < m_next_frame_time){ - return; // skip + return; // skip frame } // Advance by fixed intervals + // Next frame time is anchored relative to the first frame's time, with increments by a multiple of m_frame_interval, + // instead of being relative to the current frame's time. This prevents timing drift. + + // If there is a massive jump in time (e.g. the stream pauses for 5 seconds), + // the while loop advances the schedule multiple times until it is once again ahead of the + // current timestamp. If this happens, there will be a matching gap in the saved frames. + // We handle this gap by duplicating frames in the save() function, so that we maintain a constant frame rate. while (m_next_frame_time <= frame->timestamp){ m_next_frame_time += std::chrono::microseconds(m_frame_interval); } @@ -362,7 +369,7 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ // 2. Loop through frames for (CompressedVideoFrame frame : frames) { - // Insert dummy frames if there is a gap due to dropping frames. + // Insert duplicate frames if there is a gap due to dropping frames. // Because VideoWriter can only handle a fixed frame rate. while (current_timeline + m_frame_interval < frame.timestamp) { // Decompress last known good frame and write again From 086aa36ed741c0e5e872e0bacc3110e8f6bd3ed1 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 3 Feb 2026 13:28:39 -0800 Subject: [PATCH 16/20] use Thread wrapper to replace std::thread --- .../Recording/StreamHistoryTracker_SaveFrames.cpp | 2 +- .../Recording/StreamHistoryTracker_SaveFrames.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index cf9de41552..289d0f9c22 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -218,7 +218,7 @@ StreamHistoryTracker::StreamHistoryTracker( , m_target_fps(get_target_fps()) , m_frame_interval(1000000 / m_target_fps) { - m_worker = std::thread(&StreamHistoryTracker::worker_loop, this); + m_worker = Thread([this]{ worker_loop(); }); } void StreamHistoryTracker::set_window(std::chrono::seconds window){ diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index 2f5fe7368b..a01816b417 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -13,6 +13,7 @@ #include #include "Common/Cpp/Concurrency/Mutex.h" #include "Common/Cpp/Concurrency/ConditionVariable.h" +#include "Common/Cpp/Concurrency/Thread.h" namespace PokemonAutomation{ @@ -82,7 +83,7 @@ class StreamHistoryTracker{ std::deque m_compressed_frames; WallClock m_next_frame_time; - std::thread m_worker; + Thread m_worker; std::atomic m_stopping{false}; // Queue for the worker thread From 8b75d36ce1f31342a98785b174e16f084157daba Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 3 Feb 2026 16:36:33 -0800 Subject: [PATCH 17/20] remove unused headers --- .../StreamHistoryTracker_SaveFrames.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index 289d0f9c22..543d4800ab 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -8,16 +8,15 @@ */ #include -// #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Logging/AbstractLogger.h" #include "Common/Cpp/Concurrency/SpinLock.h" From 0680fe475fd45d52bf4d28bf6a4d0c82c87cc752 Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 3 Feb 2026 22:16:52 -0800 Subject: [PATCH 18/20] insert duplicate frames based on frame index instead of incrementing a fixed frame interval. --- .../StreamHistoryTracker_SaveFrames.cpp | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index 543d4800ab..b1d52d31c1 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -364,29 +364,35 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ } std::vector last_good_buffer = frames[0].compressed_frame; - WallClock current_timeline = frames[0].timestamp; + + size_t frames_inserted = 0; + WallClock start_time = frames[0].timestamp; // 2. Loop through frames for (CompressedVideoFrame frame : frames) { // Insert duplicate frames if there is a gap due to dropping frames. // Because VideoWriter can only handle a fixed frame rate. - while (current_timeline + m_frame_interval < frame.timestamp) { + + // calculates the frame index that this timestamp SHOULD be at + double elapsed = std::chrono::duration_cast(frame.timestamp - start_time).count(); + double interval = std::chrono::duration_cast(m_frame_interval).count(); + size_t target_frame_index = (size_t)std::round(elapsed/interval); + // fill the gap with duplicate frames until we reach the target index + while (frames_inserted < target_frame_index) { // Decompress last known good frame and write again QImage img = decompress_video_frame(last_good_buffer); cv::Mat mat(height, width, CV_8UC3, (void*)img.bits(), img.bytesPerLine()); writer.write(mat); - - current_timeline += m_frame_interval; + frames_inserted++; } + // 3. decompress frame and write to video QImage img = decompress_video_frame(frame.compressed_frame); - cv::Mat mat(height, width, CV_8UC3, (void*)img.bits(), img.bytesPerLine()); - last_good_buffer = frame.compressed_frame; - current_timeline = frame.timestamp; - - // 3. Write to video writer.write(mat); + + last_good_buffer = frame.compressed_frame; + frames_inserted++; } // Writer automatically releases when going out of scope From b681811259c2dc83ff202f3dbc466f292a77d7da Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 3 Feb 2026 22:29:45 -0800 Subject: [PATCH 19/20] add logging --- .../Recording/StreamHistoryTracker_SaveFrames.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index b1d52d31c1..e538b0ee50 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -344,7 +344,7 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ frames = m_compressed_frames; } - cout << "frames.size(): " << frames.size() << endl; + m_logger.log("Total frames to save: " + std::to_string(frames.size())); if (frames.empty()) return false; @@ -353,7 +353,7 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ int width = first_img.width(); int height = first_img.height(); - cout << width << endl; + m_logger.log("Frame size: " + std::to_string(width) + " x " + std::to_string(height)); // 1. Initialize VideoWriter (e.g., MP4 with 30 FPS) cv::VideoWriter writer(filename, cv::VideoWriter::fourcc('m', 'p', '4', 'v'), @@ -365,11 +365,16 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ std::vector last_good_buffer = frames[0].compressed_frame; + size_t frame_index = 0; size_t frames_inserted = 0; WallClock start_time = frames[0].timestamp; // 2. Loop through frames for (CompressedVideoFrame frame : frames) { + if (frame_index % 100 == 0) { + m_logger.log("Saving frame " + std::to_string(frame_index) + " / " + std::to_string(frames.size())); + } + // Insert duplicate frames if there is a gap due to dropping frames. // Because VideoWriter can only handle a fixed frame rate. @@ -393,6 +398,7 @@ bool StreamHistoryTracker::save(const std::string& filename) const{ last_good_buffer = frame.compressed_frame; frames_inserted++; + frame_index++; } // Writer automatically releases when going out of scope From 4149abcfc93213b1fcbc574535525aa7fc4f43e4 Mon Sep 17 00:00:00 2001 From: jw098 Date: Fri, 6 Feb 2026 16:52:27 -0800 Subject: [PATCH 20/20] fix initialization of m_next_frame_time --- .../Recording/StreamHistoryTracker_SaveFrames.cpp | 3 ++- .../Recording/StreamHistoryTracker_SaveFrames.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp index e538b0ee50..6485092ca2 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.cpp @@ -216,6 +216,7 @@ StreamHistoryTracker::StreamHistoryTracker( , m_has_video(has_video) , m_target_fps(get_target_fps()) , m_frame_interval(1000000 / m_target_fps) + , m_next_frame_time(WallClock::min()) { m_worker = Thread([this]{ worker_loop(); }); } @@ -249,7 +250,7 @@ void StreamHistoryTracker::on_frame(std::shared_ptr frame){ // cout << "on_frame() = " << m_frames.size() << endl; // Initialize on first frame - if (m_next_frame_time == WallClock{}){ + if (m_next_frame_time == WallClock::min()){ m_next_frame_time = frame->timestamp; } diff --git a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h index a01816b417..f530550ded 100644 --- a/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h +++ b/SerialPrograms/Source/CommonFramework/Recording/StreamHistoryTracker_SaveFrames.h @@ -75,13 +75,13 @@ class StreamHistoryTracker{ const bool m_has_video; size_t m_target_fps; std::chrono::microseconds m_frame_interval; + WallClock m_next_frame_time; // We use shared_ptr here so it's fast to snapshot when we need to copy // everything asynchronously. std::deque> m_audio; std::deque> m_frames; std::deque m_compressed_frames; - WallClock m_next_frame_time; Thread m_worker; std::atomic m_stopping{false};