LadybirdBrowser/ladybird
1
1
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-01-07 19:57:45 +03:00
Code Issues Packages Projects Releases Wiki Activity

LibWeb: Begin tracking HTMLMediaElement playback positions

Browse Source 
There are several playback positions to be tracked, depending on the
state of the media element.
This commit is contained in:
Timothy Flynn 2023-04-10 10:07:23 -04:00 committed by Linus Groh
parent 6b51c3c1ce
commit 3d9106b1b5
Notes: sideshowbarker 2024-07-17 03:03:37 +09:00 
Author: https://github.com/trflynn89
Commit: https://github.com/SerenityOS/serenity/commit/3d9106b1b5
Pull-request: https://github.com/SerenityOS/serenity/pull/18281
Reviewed-by: https://github.com/linusg
5 changed files with 195 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp
View File

@ -124,6 +124,55 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::load()
return {};
}
// https://html.spec.whatwg.org/multipage/media.html#dom-media-currenttime
double HTMLMediaElement::current_time() const
{
// The currentTime attribute must, on getting, return the media element's default playback start position, unless that is zero,
// in which case it must return the element's official playback position. The returned value must be expressed in seconds.
if (m_default_playback_start_position != 0)
return m_default_playback_start_position;
return m_official_playback_position;
}
// https://html.spec.whatwg.org/multipage/media.html#dom-media-currenttime
void HTMLMediaElement::set_current_time(double current_time)
{
// On setting, if the media element's readyState is HAVE_NOTHING, then it must set the media element's default playback start
// position to the new value; otherwise, it must set the official playback position to the new value and then seek to the new
// value. The new value must be interpreted as being in seconds.
if (m_ready_state == ReadyState::HaveNothing) {
m_default_playback_start_position = current_time;
} else {
m_official_playback_position = current_time;
// FIXME: Seek to the provided position.
}
}
// https://html.spec.whatwg.org/multipage/media.html#time-marches-on#playing-the-media-resource:current-playback-position-13
void HTMLMediaElement::set_current_playback_position(double playback_position)
{
// When the current playback position of a media element changes (e.g. due to playback or seeking), the user agent must
// run the time marches on steps. To support use cases that depend on the timing accuracy of cue event firing, such as
// synchronizing captions with shot changes in a video, user agents should fire cue events as close as possible to their
// position on the media timeline, and ideally within 20 milliseconds. If the current playback position changes while the
// steps are running, then the user agent must wait for the steps to complete, and then must immediately rerun the steps.
// These steps are thus run as often as possible or needed.
// FIXME: Detect "the current playback position changes while the steps are running".
m_current_playback_position = playback_position;
// FIXME: Regarding the official playback position, the spec states:
//
// Any time the user agent provides a stable state, the official playback position must be set to the current playback position.
// https://html.spec.whatwg.org/multipage/media.html#time-marches-on#playing-the-media-resource:official-playback-position-2
//
// We do not currently have a means to track a "stable state", so for now, keep the official playback position
// in sync with the current playback position.
m_official_playback_position = m_current_playback_position;
time_marches_on();
}
// https://html.spec.whatwg.org/multipage/media.html#dom-media-duration
double HTMLMediaElement::duration() const
{
@ -244,10 +293,21 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::load_element()
}
// FIXME: 7. If seeking is true, set it to false.
// FIXME: 8. Set the current playback position to 0.
// Set the official playback position to 0.
// If this changed the official playback position, then queue a media element task given the media element to fire an
// event named timeupdate at the media element.
// 8. Set the current playback position to 0.
m_current_playback_position = 0;
if (m_official_playback_position != 0) {
// Set the official playback position to 0.
m_official_playback_position = 0;
// If this changed the official playback position, then queue a media element task given the media element to fire an
// event named timeupdate at the media element.
queue_a_media_element_task([this] {
dispatch_time_update_event().release_value_but_fixme_should_propagate_errors();
});
}
// FIXME: 9. Set the timeline offset to Not-a-Number (NaN).
// 10. Update the duration attribute to Not-a-Number (NaN).
@ -671,7 +731,10 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void()>
// FIXME: 1. Establish the media timeline for the purposes of the current playback position and the earliest possible position, based on the media data.
// FIXME: 2. Update the timeline offset to the date and time that corresponds to the zero time in the media timeline established in the previous step,
// if any. If no explicit time and date is given by the media resource, the timeline offset must be set to Not-a-Number (NaN).
// FIXME: 3. Set the current playback position and the official playback position to the earliest possible position.
// 3. Set the current playback position and the official playback position to the earliest possible position.
m_current_playback_position = 0;
m_official_playback_position = 0;
// 4. Update the duration attribute with the time of the last frame of the resource, if known, on the media timeline established above. If it is
// not known (e.g. a stream that is in principle infinite), update the duration attribute to the value positive Infinity.
@ -694,9 +757,18 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void()>
// 6. Set the readyState attribute to HAVE_METADATA.
set_ready_state(ReadyState::HaveMetadata);
// FIXME: 7. Let jumped be false.
// FIXME: 8. If the media element's default playback start position is greater than zero, then seek to that time, and let jumped be true.
// FIXME: 9. Let the media element's default playback start position be zero.
// 7. Let jumped be false.
[[maybe_unused]] auto jumped = false;
// 8. If the media element's default playback start position is greater than zero, then seek to that time, and let jumped be true.
if (m_default_playback_start_position > 0) {
// FIXME: Seek to the default playback position.
jumped = true;
}
// 9. Let the media element's default playback start position be zero.
m_default_playback_start_position = 0;
// FIXME: 10. Let the initial playback position be zero.
// FIXME: 11. If either the media resource or the URL of the current media resource indicate a particular start time, then set the initial playback
// position to that time and, if jumped is still false, seek to that time.
@ -939,7 +1011,8 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::pause_element()
reject_pending_play_promises<WebIDL::AbortError>(promises, "Media playback was paused"_fly_string.release_value_but_fixme_should_propagate_errors());
});
// FIXME: 4. Set the official playback position to the current playback position.
// 4. Set the official playback position to the current playback position.
m_official_playback_position = m_current_playback_position;
}
return {};
@ -985,6 +1058,87 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::dispatch_time_update_event()
return {};
}
// https://html.spec.whatwg.org/multipage/media.html#time-marches-on
void HTMLMediaElement::time_marches_on(TimeMarchesOnReason reason)
{
// FIXME: 1. Let current cues be a list of cues, initialized to contain all the cues of all the hidden or showing text tracks
// of the media element (not the disabled ones) whose start times are less than or equal to the current playback
// position and whose end times are greater than the current playback position.
// FIXME: 2. Let other cues be a list of cues, initialized to contain all the cues of hidden and showing text tracks of the
// media element that are not present in current cues.
// FIXME: 3. Let last time be the current playback position at the time this algorithm was last run for this media element,
// if this is not the first time it has run.
// FIXME: 4. If the current playback position has, since the last time this algorithm was run, only changed through its usual
// monotonic increase during normal playback, then let missed cues be the list of cues in other cues whose start times
// are greater than or equal to last time and whose end times are less than or equal to the current playback position.
// Otherwise, let missed cues be an empty list.
// FIXME: 5. Remove all the cues in missed cues that are also in the media element's list of newly introduced cues, and then
// empty the element's list of newly introduced cues.
// 6. If the time was reached through the usual monotonic increase of the current playback position during normal
// playback, and if the user agent has not fired a timeupdate event at the element in the past 15 to 250ms and is
// not still running event handlers for such an event, then the user agent must queue a media element task given
// the media element to fire an event named timeupdate at the element. (In the other cases, such as explicit seeks,
// relevant events get fired as part of the overall process of changing the current playback position.)
if (reason == TimeMarchesOnReason::NormalPlayback && !m_running_time_update_event_handler) {
auto dispatch_event = true;
if (m_last_time_update_event_time.has_value()) {
auto time_since_last_event = Time::now_monotonic() - *m_last_time_update_event_time;
dispatch_event = time_since_last_event.to_milliseconds() > 250;
}
if (dispatch_event) {
queue_a_media_element_task([this]() {
dispatch_time_update_event().release_value_but_fixme_should_propagate_errors();
});
}
}
// FIXME: 7. If all of the cues in current cues have their text track cue active flag set, none of the cues in other cues have
// their text track cue active flag set, and missed cues is empty, then return.
// FIXME: 8. If the time was reached through the usual monotonic increase of the current playback position during normal playback,
// and there are cues in other cues that have their text track cue pause-on-exit flag set and that either have their
// text track cue active flag set or are also in missed cues, then immediately pause the media element.
// FIXME: 9. Let events be a list of tasks, initially empty. Each task in this list will be associated with a text track, a
// text track cue, and a time, which are used to sort the list before the tasks are queued.
//
// Let affected tracks be a list of text tracks, initially empty.
//
// When the steps below say to prepare an event named event for a text track cue target with a time time, the user
// agent must run these steps:
// 1. Let track be the text track with which the text track cue target is associated.
// 2. Create a task to fire an event named event at target.
// 3. Add the newly created task to events, associated with the time time, the text track track, and the text
// track cue target.
// 4. Add track to affected tracks.
// FIXME: 10. For each text track cue in missed cues, prepare an event named enter for the TextTrackCue object with the text
// track cue start time.
// FIXME: 11. For each text track cue in other cues that either has its text track cue active flag set or is in missed cues,
// prepare an event named exit for the TextTrackCue object with the later of the text track cue end time and the
/// text track cue start time.
// FIXME: 12. For each text track cue in current cues that does not have its text track cue active flag set, prepare an event
// named enter for the TextTrackCue object with the text track cue start time.
// FIXME: 13. Sort the tasks in events in ascending time order (tasks with earlier times first).
//
// Further sort tasks in events that have the same time by the relative text track cue order of the text track cues
// associated with these tasks.
//
// Finally, sort tasks in events that have the same time and same text track cue order by placing tasks that fire
// enter events before those that fire exit events.
// FIXME: 14. Queue a media element task given the media element for each task in events, in list order.
// FIXME: 15. Sort affected tracks in the same order as the text tracks appear in the media element's list of text tracks, and
// remove duplicates.
// FIXME: 16. For each text track in affected tracks, in the list order, queue a media element task given the media element to
// fire an event named cuechange at the TextTrack object, and, if the text track has a corresponding track element,
// to then fire an event named cuechange at the track element as well.
// FIXME: 17. Set the text track cue active flag of all the cues in the current cues, and unset the text track cue active flag
// of all the cues in the other cues.
// FIXME: 18. Run the rules for updating the text track rendering of each of the text tracks in affected tracks that are showing,
// providing the text track's text track language as the fallback language if it is not the empty string. For example,
// for text tracks based on WebVTT, the rules for updating the display of WebVTT text tracks.
}
// https://html.spec.whatwg.org/multipage/media.html#take-pending-play-promises
JS::MarkedVector<JS::NonnullGCPtr<WebIDL::Promise>> HTMLMediaElement::take_pending_play_promises()
{

20
Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h
View File

@ -47,6 +47,11 @@ public:
ReadyState ready_state() const { return m_ready_state; }
WebIDL::ExceptionOr<void> load();
double current_time() const;
void set_current_time(double);
void set_current_playback_position(double);
double duration() const;
bool paused() const { return m_paused; }
WebIDL::ExceptionOr<JS::NonnullGCPtr<JS::Promise>> play();
@ -90,6 +95,12 @@ private:
WebIDL::ExceptionOr<void> dispatch_time_update_event();
enum class TimeMarchesOnReason {
NormalPlayback,
Other,
};
void time_marches_on(TimeMarchesOnReason = TimeMarchesOnReason::NormalPlayback);
JS::MarkedVector<JS::NonnullGCPtr<WebIDL::Promise>> take_pending_play_promises();
void resolve_pending_play_promises(ReadonlySpan<JS::NonnullGCPtr<WebIDL::Promise>> promises);
void reject_pending_play_promises(ReadonlySpan<JS::NonnullGCPtr<WebIDL::Promise>> promises, JS::NonnullGCPtr<WebIDL::DOMException> error);
@ -114,6 +125,15 @@ private:
ReadyState m_ready_state { ReadyState::HaveNothing };
bool m_first_data_load_event_since_load_start { false };
// https://html.spec.whatwg.org/multipage/media.html#current-playback-position
double m_current_playback_position { 0 };
// https://html.spec.whatwg.org/multipage/media.html#official-playback-position
double m_official_playback_position { 0 };
// https://html.spec.whatwg.org/multipage/media.html#default-playback-start-position
double m_default_playback_start_position { 0 };
// https://html.spec.whatwg.org/multipage/media.html#dom-media-duration
double m_duration { NAN };

1
Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl
View File

@ -30,6 +30,7 @@ interface HTMLMediaElement : HTMLElement {
readonly attribute unsigned short readyState;
// playback state
attribute double currentTime;
readonly attribute unrestricted double duration;
readonly attribute boolean paused;
[Reflect, CEReactions] attribute boolean autoplay;

9
Userland/Libraries/LibWeb/HTML/VideoTrack.cpp
View File

@ -5,6 +5,7 @@
*/
#include <AK/IDAllocator.h>
#include <AK/Time.h>
#include <LibGfx/Bitmap.h>
#include <LibJS/Runtime/Realm.h>
#include <LibJS/Runtime/VM.h>
@ -31,6 +32,9 @@ VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> medi
m_playback_manager->on_video_frame = [this](auto frame) {
if (is<HTMLVideoElement>(*m_media_element))
verify_cast<HTMLVideoElement>(*m_media_element).set_current_frame({}, move(frame));
auto playback_position_ms = static_cast<double>(position().to_milliseconds());
m_media_element->set_current_playback_position(playback_position_ms / 1000.0);
};
m_playback_manager->on_decoder_error = [](auto) {
@ -77,6 +81,11 @@ void VideoTrack::pause_video(Badge<HTMLVideoElement>)
m_playback_manager->pause_playback();
}
Time VideoTrack::position() const
{
return m_playback_manager->current_playback_time();
}
Time VideoTrack::duration() const
{
return m_playback_manager->selected_video_track().video_data().duration;

2
Userland/Libraries/LibWeb/HTML/VideoTrack.h
View File

@ -25,7 +25,9 @@ public:
void play_video(Badge<HTMLVideoElement>);
void pause_video(Badge<HTMLVideoElement>);
Time position() const;
Time duration() const;
u64 pixel_width() const;
u64 pixel_height() const;