LadybirdBrowser/ladybird
1
1
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-09-21 02:08:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

LibVideo: Delay playback errors by their sample timestamps when we can

Browse Source 
If we encounter an error in the video decoder, we can set a timestamp
for the error item in the queue so that it will display the error only
when the frame that caused the error would have been displayed.
This commit is contained in:
Zaggy1024 2023-02-12 00:50:30 -06:00 committed by Andreas Kling
parent 2a228e8a6e
commit 5da0c6f916
Notes: sideshowbarker 2024-07-17 00:29:08 +09:00 
Author: https://github.com/Zaggy1024
Commit: https://github.com/SerenityOS/serenity/commit/5da0c6f916
Pull-request: https://github.com/SerenityOS/serenity/pull/17448
2 changed files with 49 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
Userland/Libraries/LibVideo/PlaybackManager.cpp
View File

@ -161,23 +161,27 @@ bool PlaybackManager::decode_and_queue_one_sample()
auto start_time = Time::now_monotonic();
#endif
#define TRY_OR_ENQUEUE_ERROR(expression) \
({ \
auto&& _temporary_result = ((expression)); \
if (_temporary_result.is_error()) { \
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Enqueued decoder error: {}", _temporary_result.error().string_literal()); \
m_frame_queue->enqueue(FrameQueueItem::error_marker(_temporary_result.release_error())); \
return false; \
} \
static_assert(!::AK::Detail::IsLvalueReference<decltype(_temporary_result.release_value())>, \
"Do not return a reference from a fallible expression"); \
_temporary_result.release_value(); \
auto enqueue_error = [&](DecoderError&& error, Time timestamp) {
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Enqueued decoder error: {}", error.string_literal());
m_frame_queue->enqueue(FrameQueueItem::error_marker(move(error), timestamp));
};
#define TRY_OR_ENQUEUE_ERROR(expression, timestamp) \
({ \
auto&& _temporary_result = (expression); \
if (_temporary_result.is_error()) { \
enqueue_error(_temporary_result.release_error(), (timestamp)); \
return false; \
} \
static_assert(!::AK::Detail::IsLvalueReference<decltype(_temporary_result.release_value())>, \
"Do not return a reference from a fallible expression"); \
_temporary_result.release_value(); \
})
auto frame_sample = TRY_OR_ENQUEUE_ERROR(m_demuxer->get_next_video_sample_for_track(m_selected_video_track));
auto frame_sample = TRY_OR_ENQUEUE_ERROR(m_demuxer->get_next_video_sample_for_track(m_selected_video_track), Time::min());
OwnPtr<VideoFrame> decoded_frame = nullptr;
while (!decoded_frame) {
TRY_OR_ENQUEUE_ERROR(m_decoder->receive_sample(frame_sample->data()));
TRY_OR_ENQUEUE_ERROR(m_decoder->receive_sample(frame_sample->data()), frame_sample->timestamp());
while (true) {
auto frame_result = m_decoder->get_decoded_frame();
@ -186,7 +190,7 @@ bool PlaybackManager::decode_and_queue_one_sample()
if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
break;
dispatch_decoder_error(frame_result.release_error());
enqueue_error(frame_result.release_error(), frame_sample->timestamp());
return false;
}
@ -214,7 +218,7 @@ bool PlaybackManager::decode_and_queue_one_sample()
break;
}
auto bitmap = TRY_OR_ENQUEUE_ERROR(decoded_frame->to_bitmap());
auto bitmap = TRY_OR_ENQUEUE_ERROR(decoded_frame->to_bitmap(), frame_sample->timestamp());
m_frame_queue->enqueue(FrameQueueItem::frame(bitmap, frame_sample->timestamp()));
#if PLAYBACK_MANAGER_DEBUG
@ -317,7 +321,7 @@ private:
future_frame_item.emplace(manager().m_frame_queue->dequeue());
manager().m_decode_timer->start(0);
if (future_frame_item->is_error() || future_frame_item->timestamp() >= current_time()) {
if (future_frame_item->timestamp() >= current_time() || future_frame_item->timestamp() == FrameQueueItem::no_timestamp) {
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Should present frame, future {} is error or after {}ms", future_frame_item->debug_string(), current_time().to_milliseconds());
should_present_frame = true;
break;
@ -468,7 +472,7 @@ private:
ErrorOr<void> on_enter() override
{
auto earliest_available_sample = manager().m_last_present_in_media_time;
if (manager().m_next_frame.has_value() && manager().m_next_frame->is_frame()) {
if (manager().m_next_frame.has_value() && manager().m_next_frame->timestamp() != FrameQueueItem::no_timestamp) {
earliest_available_sample = min(earliest_available_sample, manager().m_next_frame->timestamp());
}
auto keyframe_timestamp = manager().seek_demuxer_to_most_recent_keyframe(m_target_timestamp, earliest_available_sample);
@ -504,31 +508,22 @@ private:
auto item = manager().m_frame_queue->dequeue();
manager().m_decode_timer->start(0);
if (item.is_error()) {
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Encountered error while seeking: {}", item.error().description());
manager().dispatch_decoder_error(item.release_error());
return {};
}
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Dequeuing frame at {}ms and comparing to seek target {}ms", item.timestamp().to_milliseconds(), m_target_timestamp.to_milliseconds());
if (item.timestamp() > m_target_timestamp || item.timestamp() == FrameQueueItem::no_timestamp) {
// Fast seeking will result in an equal timestamp, so we can exit as soon as we see the next frame.
if (manager().m_next_frame.has_value()) {
manager().m_last_present_in_media_time = m_target_timestamp;
if (item.is_frame()) {
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Dequeuing frame at {}ms and comparing to seek target {}ms", item.timestamp().to_milliseconds(), m_target_timestamp.to_milliseconds());
if (item.timestamp() > m_target_timestamp) {
// Fast seeking will result in an equal timestamp, so we can exit as soon as we see the next frame.
if (manager().m_next_frame.has_value()) {
manager().dispatch_new_frame(manager().m_next_frame.release_value().bitmap());
manager().m_last_present_in_media_time = m_target_timestamp;
}
manager().m_next_frame.emplace(item);
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Exiting seek to {} state at {}ms", m_playing ? "Playing" : "Paused", manager().m_last_present_in_media_time.to_milliseconds());
return exit_seek();
if (manager().dispatch_frame_queue_item(manager().m_next_frame.release_value()))
return {};
}
manager().m_next_frame.emplace(item);
continue;
}
VERIFY_NOT_REACHED();
manager().m_next_frame.emplace(item);
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Exiting seek to {} state at {}ms", m_playing ? "Playing" : "Paused", manager().m_last_present_in_media_time.to_milliseconds());
return exit_seek();
}
manager().m_next_frame.emplace(item);
}
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Frame queue is empty while seeking, waiting for buffer fill.");

29
Userland/Libraries/LibVideo/PlaybackManager.h
View File

@ -25,6 +25,8 @@
namespace Video {
struct FrameQueueItem {
static constexpr Time no_timestamp = Time::min();
enum class Type {
Frame,
Error,
@ -35,14 +37,14 @@ struct FrameQueueItem {
return FrameQueueItem(move(bitmap), timestamp);
}
static FrameQueueItem error_marker(DecoderError&& error)
static FrameQueueItem error_marker(DecoderError&& error, Time timestamp)
{
return FrameQueueItem(move(error));
return FrameQueueItem(move(error), timestamp);
}
bool is_frame() const { return m_data.has<FrameData>(); }
RefPtr<Gfx::Bitmap> bitmap() const { return m_data.get<FrameData>().bitmap; }
Time timestamp() const { return m_data.get<FrameData>().timestamp; }
bool is_frame() const { return m_data.has<RefPtr<Gfx::Bitmap>>(); }
RefPtr<Gfx::Bitmap> bitmap() const { return m_data.get<RefPtr<Gfx::Bitmap>>(); }
Time timestamp() const { return m_timestamp; }
bool is_error() const { return m_data.has<DecoderError>(); }
DecoderError const& error() const { return m_data.get<DecoderError>(); }
@ -56,27 +58,26 @@ struct FrameQueueItem {
DeprecatedString debug_string() const
{
if (is_error())
return error().string_literal();
return DeprecatedString::formatted("{} at {}ms", error().string_literal(), timestamp().to_milliseconds());
return DeprecatedString::formatted("frame at {}ms", timestamp().to_milliseconds());
}
private:
struct FrameData {
RefPtr<Gfx::Bitmap> bitmap;
Time timestamp;
};
FrameQueueItem(RefPtr<Gfx::Bitmap> bitmap, Time timestamp)
: m_data(FrameData { move(bitmap), timestamp })
: m_data(move(bitmap))
, m_timestamp(timestamp)
{
VERIFY(m_timestamp != no_timestamp);
}
FrameQueueItem(DecoderError&& error)
FrameQueueItem(DecoderError&& error, Time timestamp)
: m_data(move(error))
, m_timestamp(timestamp)
{
}
Variant<Empty, FrameData, DecoderError> m_data;
Variant<Empty, RefPtr<Gfx::Bitmap>, DecoderError> m_data;
Time m_timestamp;
};
static constexpr size_t FRAME_BUFFER_COUNT = 4;