LibMedia+LibWeb: Update the duration of an element based on media data

Libraries/LibMedia/AudioBlock.h

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <AK/FixedArray.h>
+#include <AK/Math.h>
 #include <AK/Time.h>
 
 namespace Media {
@@ -19,6 +20,8 @@ class AudioBlock {
     u8 channel_count() const { return m_channel_count; }
     AK::Duration timestamp() const { return m_timestamp; }
     i64 timestamp_in_samples() const { return m_timestamp_in_samples; }
+    i64 end_timestamp_in_samples() const { return Checked<i64>::saturating_add(m_timestamp_in_samples, AK::clamp_to<i64>(sample_count())); }
+    AK::Duration end_timestamp() const { return AK::Duration::from_time_units(end_timestamp_in_samples(), 1, sample_rate()); }
     Data& data() { return m_data; }
     Data const& data() const { return m_data; }
 

Libraries/LibMedia/CodedFrame.h

@@ -18,22 +18,25 @@ class CodedFrame final {
 public:
     using AuxiliaryData = Variant<CodedVideoFrameData, CodedAudioFrameData>;
 
-    CodedFrame(AK::Duration timestamp, FrameFlags flags, ByteBuffer&& data, AuxiliaryData auxiliary_data)
+    CodedFrame(AK::Duration timestamp, AK::Duration duration, FrameFlags flags, ByteBuffer&& data, AuxiliaryData auxiliary_data)
         : m_timestamp(timestamp)
+        , m_duration(duration)
         , m_flags(flags)
         , m_data(move(data))
         , m_auxiliary_data(auxiliary_data)
     {
     }
 
     AK::Duration timestamp() const { return m_timestamp; }
+    AK::Duration duration() const { return m_duration; }
     FrameFlags flags() const { return m_flags; }
     bool is_keyframe() const { return has_flag(m_flags, FrameFlags::Keyframe); }
     ByteBuffer const& data() const { return m_data; }
     AuxiliaryData const& auxiliary_data() const { return m_auxiliary_data; }
 
 private:
     AK::Duration m_timestamp;
+    AK::Duration m_duration;
     FrameFlags m_flags;
     ByteBuffer m_data;
     AuxiliaryData m_auxiliary_data;

Libraries/LibMedia/Containers/Matroska/Document.h

@@ -139,6 +139,8 @@ class TrackEntry : public RefCounted<TrackEntry> {
     void set_seek_pre_roll(u64 seek_pre_roll) { m_seek_pre_roll = seek_pre_roll; }
     u64 timestamp_offset() const { return m_timestamp_offset; }
     void set_timestamp_offset(u64 timestamp_offset) { m_timestamp_offset = timestamp_offset; }
+    u64 default_duration() const { return m_default_duration; }
+    void set_default_duration(u64 default_duration) { m_default_duration = default_duration; }
     Optional<VideoTrack> video_track() const { return m_video_track; }
     void set_video_track(VideoTrack video_track) { m_video_track = video_track; }
     Optional<AudioTrack> audio_track() const { return m_audio_track; }
@@ -157,6 +159,7 @@ class TrackEntry : public RefCounted<TrackEntry> {
     u64 m_codec_delay { 0 };
     u64 m_seek_pre_roll { 0 };
     u64 m_timestamp_offset { 0 };
+    u64 m_default_duration { 0 };
     Optional<VideoTrack> m_video_track;
     Optional<AudioTrack> m_audio_track;
 };
@@ -174,6 +177,8 @@ class Block {
     void set_track_number(u64 track_number) { m_track_number = track_number; }
     AK::Duration timestamp() const { return m_timestamp; }
     void set_timestamp(AK::Duration timestamp) { m_timestamp = timestamp; }
+    Optional<AK::Duration> duration() const { return m_duration; }
+    void set_duration(AK::Duration duration) { m_duration = duration; }
     bool only_keyframes() const { return m_only_keyframes; }
     void set_only_keyframes(bool only_keyframes) { m_only_keyframes = only_keyframes; }
     bool invisible() const { return m_invisible; }
@@ -191,6 +196,7 @@ class Block {
 private:
     u64 m_track_number { 0 };
     AK::Duration m_timestamp { AK::Duration::zero() };
+    Optional<AK::Duration> m_duration;
     bool m_only_keyframes { false };
     bool m_invisible { false };
     Lacing m_lacing { None };

Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp

@@ -175,6 +175,10 @@ DecoderErrorOr<CodedFrame> MatroskaDemuxer::get_next_sample_for_track(Track cons
         status.frame_index = 0;
     }
 
+    VERIFY(status.block.has_value());
+
+    auto timestamp = status.block->timestamp();
+    auto duration = status.block->duration().value_or(AK::Duration::zero());
     auto flags = status.block->only_keyframes() ? FrameFlags::Keyframe : FrameFlags::None;
     auto aux_data = [&] -> CodedFrame::AuxiliaryData {
         if (track.type() == TrackType::Video) {
@@ -186,7 +190,7 @@ DecoderErrorOr<CodedFrame> MatroskaDemuxer::get_next_sample_for_track(Track cons
         VERIFY_NOT_REACHED();
     }();
     auto sample_data = DECODER_TRY_ALLOC(ByteBuffer::copy(status.block->frame(status.frame_index++)));
-    return CodedFrame(status.block->timestamp(), flags, move(sample_data), aux_data);
+    return CodedFrame(timestamp, duration, flags, move(sample_data), aux_data);
 }
 
 DecoderErrorOr<AK::Duration> MatroskaDemuxer::total_duration()

Libraries/LibMedia/Containers/Matroska/Reader.cpp

@@ -61,6 +61,7 @@ constexpr u32 TRACK_CODEC_DELAY_ID = 0x56AA;
 constexpr u32 TRACK_SEEK_PRE_ROLL_ID = 0x56BB;
 constexpr u32 TRACK_TIMESTAMP_SCALE_ID = 0x23314F;
 constexpr u32 TRACK_OFFSET_ID = 0x537F;
+constexpr u32 TRACK_DEFAULT_DURATION_ID = 0x23E383;
 constexpr u32 TRACK_VIDEO_ID = 0xE0;
 constexpr u32 TRACK_AUDIO_ID = 0xE1;
 
@@ -84,6 +85,7 @@ constexpr u32 SIMPLE_BLOCK_ID = 0xA3;
 constexpr u32 TIMESTAMP_ID = 0xE7;
 constexpr u32 BLOCK_GROUP_ID = 0xA0;
 constexpr u32 BLOCK_ID = 0xA1;
+constexpr u32 BLOCK_DURATION_ID = 0x9B;
 
 // Cues
 constexpr u32 CUES_ID = 0x1C53BB6B;
@@ -526,6 +528,10 @@ static DecoderErrorOr<NonnullRefPtr<TrackEntry>> parse_track_entry(Streamer& str
             track_entry->set_timestamp_offset(TRY_READ(streamer.read_variable_size_signed_integer()));
             dbgln_if(MATROSKA_TRACE_DEBUG, "Read Track's TrackOffset attribute: {}", track_entry->timestamp_offset());
             break;
+        case TRACK_DEFAULT_DURATION_ID:
+            track_entry->set_default_duration(TRY_READ(streamer.read_u64()));
+            dbgln_if(MATROSKA_TRACE_DEBUG, "Read Track's DefaultDuration attribute: {}", track_entry->default_duration());
+            break;
         case TRACK_VIDEO_ID:
             track_entry->set_video_track(TRY(parse_video_track_information(streamer)));
             break;
@@ -823,9 +829,16 @@ static DecoderErrorOr<Vector<ReadonlyBytes>> parse_frames(Streamer& streamer, Bl
     return frames;
 }
 
+static void set_block_duration_to_default(Block& block, TrackEntry const& track)
+{
+    if (track.default_duration() != 0)
+        block.set_duration(AK::Duration::from_nanoseconds(AK::clamp_to<i64>(track.default_duration())));
+}
+
 static DecoderErrorOr<Block> parse_simple_block(Streamer& streamer, AK::Duration cluster_timestamp, u64 segment_timestamp_scale, TrackEntry const& track)
 {
     Block block;
+    set_block_duration_to_default(block, track);
 
     auto content_size = TRY_READ(streamer.read_variable_size_integer());
 
@@ -849,8 +862,9 @@ static DecoderErrorOr<Block> parse_simple_block(Streamer& streamer, AK::Duration
 static DecoderErrorOr<Block> parse_block_group(Streamer& streamer, AK::Duration cluster_timestamp, u64 segment_timestamp_scale, TrackEntry const& track)
 {
     Block block;
-    auto parsed_a_block = false;
+    set_block_duration_to_default(block, track);
 
+    auto parsed_a_block = false;
     TRY(parse_master_element(streamer, "BlockGroup"sv, [&](u64 element_id) -> DecoderErrorOr<IterationDecision> {
         switch (element_id) {
         case BLOCK_ID: {
@@ -873,6 +887,14 @@ static DecoderErrorOr<Block> parse_block_group(Streamer& streamer, AK::Duration
             block.set_frames(TRY(parse_frames(streamer, block.lacing(), remaining_size)));
             break;
         }
+        case BLOCK_DURATION_ID: {
+            auto duration = TRY_READ(streamer.read_u64());
+            auto duration_nanoseconds = Checked<i64>::saturating_mul(duration, segment_timestamp_scale);
+            if (track.timestamp_scale() != 1)
+                duration_nanoseconds = AK::clamp_to<i64>(static_cast<double>(duration_nanoseconds) * track.timestamp_scale());
+            block.set_duration(AK::Duration::from_nanoseconds(duration_nanoseconds));
+            break;
+        }
         default:
             TRY_READ(streamer.read_unknown_element());
             break;

Libraries/LibMedia/FFmpeg/FFmpegDemuxer.cpp

@@ -253,6 +253,7 @@ DecoderErrorOr<CodedFrame> FFmpegDemuxer::get_next_sample_for_track(Track const&
         auto flags = (packet.flags & AV_PKT_FLAG_KEY) != 0 ? FrameFlags::Keyframe : FrameFlags::None;
         auto sample = CodedFrame(
             time_units_to_duration(packet.pts, stream.time_base),
+            time_units_to_duration(packet.duration, stream.time_base),
             flags,
             move(packet_data),
             auxiliary_data);

Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.cpp

@@ -104,14 +104,15 @@ FFmpegVideoDecoder::~FFmpegVideoDecoder()
     avcodec_free_context(&m_codec_context);
 }
 
-DecoderErrorOr<void> FFmpegVideoDecoder::receive_coded_data(AK::Duration timestamp, ReadonlyBytes coded_data)
+DecoderErrorOr<void> FFmpegVideoDecoder::receive_coded_data(AK::Duration timestamp, AK::Duration duration, ReadonlyBytes coded_data)
 {
     VERIFY(coded_data.size() < NumericLimits<int>::max());
 
     m_packet->data = const_cast<u8*>(coded_data.data());
     m_packet->size = static_cast<int>(coded_data.size());
     m_packet->pts = timestamp.to_microseconds();
     m_packet->dts = m_packet->pts;
+    m_packet->duration = duration.to_microseconds();
 
     auto result = avcodec_send_packet(m_codec_context, m_packet);
     switch (result) {
@@ -210,7 +211,8 @@ DecoderErrorOr<NonnullOwnPtr<VideoFrame>> FFmpegVideoDecoder::get_decoded_frame(
         auto size = Gfx::Size<u32> { m_frame->width, m_frame->height };
 
         auto timestamp = AK::Duration::from_microseconds(m_frame->pts);
-        auto frame = DECODER_TRY_ALLOC(SubsampledYUVFrame::try_create(timestamp, size, bit_depth, cicp, subsampling));
+        auto duration = AK::Duration::from_microseconds(m_frame->duration);
+        auto frame = DECODER_TRY_ALLOC(SubsampledYUVFrame::try_create(timestamp, duration, size, bit_depth, cicp, subsampling));
 
         for (u32 plane = 0; plane < 3; plane++) {
             VERIFY(m_frame->linesize[plane] != 0);

Libraries/LibMedia/FFmpeg/FFmpegVideoDecoder.h

@@ -20,7 +20,7 @@ class MEDIA_API FFmpegVideoDecoder final : public VideoDecoder {
     FFmpegVideoDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame);
     virtual ~FFmpegVideoDecoder() override;
 
-    virtual DecoderErrorOr<void> receive_coded_data(AK::Duration timestamp, ReadonlyBytes coded_data) override;
+    virtual DecoderErrorOr<void> receive_coded_data(AK::Duration timestamp, AK::Duration duration, ReadonlyBytes coded_data) override;
     virtual void signal_end_of_stream() override;
     virtual DecoderErrorOr<NonnullOwnPtr<VideoFrame>> get_decoded_frame() override;
 

Libraries/LibMedia/PlaybackManager.cpp

@@ -84,7 +84,7 @@ DecoderErrorOr<NonnullRefPtr<PlaybackManager>> PlaybackManager::try_create(Reado
 
     auto playback_manager = DECODER_TRY_ALLOC(adopt_nonnull_ref_or_enomem(new (nothrow) PlaybackManager(demuxer, weak_playback_manager, time_provider, move(supported_video_tracks), move(supported_video_track_datas), audio_sink, move(supported_audio_tracks), move(supported_audio_track_datas))));
     weak_playback_manager->m_manager = playback_manager;
-    playback_manager->set_up_error_handlers();
+    playback_manager->set_up_data_providers();
     playback_manager->m_handler = DECODER_TRY_ALLOC(try_make<PausedStateHandler>(*playback_manager));
     return playback_manager;
 }
@@ -106,15 +106,23 @@ PlaybackManager::~PlaybackManager()
     m_weak_wrapper->revoke();
 }
 
-void PlaybackManager::set_up_error_handlers()
+void PlaybackManager::set_up_data_providers()
 {
+    m_duration = m_demuxer->total_duration().value_or(AK::Duration::zero());
+
     for (auto const& video_track_data : m_video_track_datas) {
         video_track_data.provider->set_error_handler([weak_self = m_weak_wrapper](DecoderError&& error) {
             auto self = weak_self->take_strong();
             if (!self)
                 return;
             self->dispatch_error(move(error));
         });
+        video_track_data.provider->set_frame_end_time_handler([weak_self = m_weak_wrapper](AK::Duration time) {
+            auto self = weak_self->take_strong();
+            if (!self)
+                return;
+            self->check_for_duration_change(time);
+        });
     }
 
     for (auto const& audio_track_data : m_audio_track_datas) {
@@ -124,9 +132,24 @@ void PlaybackManager::set_up_error_handlers()
                 return;
             self->dispatch_error(move(error));
         });
+        audio_track_data.provider->set_block_end_time_handler([weak_self = m_weak_wrapper](AK::Duration time) {
+            auto self = weak_self->take_strong();
+            if (!self)
+                return;
+            self->check_for_duration_change(time);
+        });
     }
 }
 
+void PlaybackManager::check_for_duration_change(AK::Duration duration)
+{
+    if (m_duration >= duration)
+        return;
+    m_duration = duration;
+    if (on_duration_change)
+        on_duration_change(m_duration);
+}
+
 void PlaybackManager::dispatch_error(DecoderError&& error)
 {
     if (m_is_in_error_state)
@@ -138,7 +161,7 @@ void PlaybackManager::dispatch_error(DecoderError&& error)
 
 AK::Duration PlaybackManager::duration() const
 {
-    return m_demuxer->total_duration().value_or(AK::Duration::zero());
+    return m_duration;
 }
 
 Optional<Track> PlaybackManager::preferred_video_track()

Libraries/LibMedia/PlaybackManager.h

@@ -78,6 +78,7 @@ class MEDIA_API PlaybackManager final : public AtomicRefCounted<PlaybackManager>
     void set_volume(double);
 
     Function<void()> on_playback_state_change;
+    Function<void(AK::Duration)> on_duration_change;
     Function<void(DecoderError&&)> on_error;
 
 private:
@@ -119,7 +120,8 @@ class MEDIA_API PlaybackManager final : public AtomicRefCounted<PlaybackManager>
 
     PlaybackManager(NonnullRefPtr<MutexedDemuxer> const&, NonnullRefPtr<WeakPlaybackManager> const&, NonnullRefPtr<MediaTimeProvider> const&, VideoTracks&&, VideoTrackDatas&&, RefPtr<AudioMixingSink> const&, AudioTracks&&, AudioTrackDatas&&);
 
-    void set_up_error_handlers();
+    void set_up_data_providers();
+    void check_for_duration_change(AK::Duration);
     void dispatch_error(DecoderError&&);
 
     VideoTrackData& get_video_data_for_track(Track const& track);
@@ -135,6 +137,7 @@ class MEDIA_API PlaybackManager final : public AtomicRefCounted<PlaybackManager>
     NonnullRefPtr<WeakPlaybackManager> m_weak_wrapper;
 
     NonnullRefPtr<MediaTimeProvider> m_time_provider;
+    AK::Duration m_duration;
 
     VideoTracks m_video_tracks;
     VideoTrackDatas m_video_track_datas;

Libraries/LibMedia/Providers/AudioDataProvider.cpp

@@ -53,6 +53,11 @@ void AudioDataProvider::set_error_handler(ErrorHandler&& handler)
     m_thread_data->set_error_handler(move(handler));
 }
 
+void AudioDataProvider::set_block_end_time_handler(BlockEndTimeHandler&& handler)
+{
+    m_thread_data->set_block_end_time_handler(move(handler));
+}
+
 void AudioDataProvider::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler)
 {
     m_thread_data->seek(timestamp, move(completion_handler));
@@ -75,6 +80,11 @@ void AudioDataProvider::ThreadData::set_error_handler(ErrorHandler&& handler)
     m_wait_condition.broadcast();
 }
 
+void AudioDataProvider::ThreadData::set_block_end_time_handler(BlockEndTimeHandler&& handler)
+{
+    m_frame_end_time_handler = move(handler);
+}
+
 AudioBlock AudioDataProvider::retrieve_block()
 {
     auto locker = m_thread_data->take_lock();
@@ -105,6 +115,25 @@ bool AudioDataProvider::ThreadData::should_thread_exit() const
     return m_exit;
 }
 
+void AudioDataProvider::ThreadData::dispatch_block_end_time(AudioBlock const& block)
+{
+    auto end_time = block.end_timestamp();
+    m_main_thread_event_loop.deferred_invoke([self = NonnullRefPtr(*this), end_time] {
+        if (self->m_frame_end_time_handler)
+            self->m_frame_end_time_handler(end_time);
+    });
+}
+
+void AudioDataProvider::ThreadData::queue_block(AudioBlock&& block)
+{
+    // FIXME: Specify trailing samples in the demuxer, and drop them here or in the audio decoder implementation.
+
+    VERIFY(!block.is_empty());
+    dispatch_block_end_time(block);
+    m_queue.enqueue(move(block));
+    VERIFY(!m_queue.tail().is_empty());
+}
+
 void AudioDataProvider::ThreadData::flush_decoder()
 {
     m_decoder->flush();
@@ -225,9 +254,9 @@ bool AudioDataProvider::ThreadData::handle_seek()
                     m_queue.clear();
 
                     if (!last_block.is_empty())
-                        m_queue.enqueue(move(last_block));
+                        queue_block(move(last_block));
 
-                    m_queue.enqueue(move(current_block));
+                    queue_block(move(current_block));
 
                     resolve_seek(seek_id);
                     return true;
@@ -306,20 +335,16 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
         }
 
         auto block = AudioBlock();
-        auto timestamp_result = retrieve_next_block(block);
-        if (timestamp_result.is_error()) {
-            if (timestamp_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
+        auto block_result = retrieve_next_block(block);
+        if (block_result.is_error()) {
+            if (block_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
                 break;
-            set_error_and_wait_for_seek(timestamp_result.release_error());
+            set_error_and_wait_for_seek(block_result.release_error());
             break;
         }
 
-        // FIXME: Specify trailing samples in the demuxer, and drop them here or in the audio decoder implementation.
-
         auto locker = take_lock();
-        VERIFY(!block.is_empty());
-        m_queue.enqueue(move(block));
-        VERIFY(!m_queue.tail().is_empty());
+        queue_block(move(block));
     }
 }