Skip to content

Unbuffered media decoding [WIP] #6921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+805 −473
Draft

Unbuffered media decoding [WIP] #6921

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions Libraries/LibMedia/Audio/FFmpegLoader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

namespace Audio {

static constexpr int BUFFER_MAX_PROBE_SIZE = 64 * KiB;
[[maybe_unused]] static constexpr int BUFFER_MAX_PROBE_SIZE = 64 * KiB;

FFmpegLoaderPlugin::FFmpegLoaderPlugin(NonnullOwnPtr<SeekableStream> stream, NonnullOwnPtr<Media::FFmpeg::FFmpegIOContext> io_context)
: LoaderPlugin(move(stream))
Expand All @@ -37,13 +37,13 @@ FFmpegLoaderPlugin::~FFmpegLoaderPlugin()
avformat_close_input(&m_format_context);
}

ErrorOr<NonnullOwnPtr<LoaderPlugin>> FFmpegLoaderPlugin::create(NonnullOwnPtr<SeekableStream> stream)
{
auto io_context = TRY(Media::FFmpeg::FFmpegIOContext::create(*stream));
auto loader = make<FFmpegLoaderPlugin>(move(stream), move(io_context));
TRY(loader->initialize());
return loader;
}
// ErrorOr<NonnullOwnPtr<LoaderPlugin>> FFmpegLoaderPlugin::create(NonnullOwnPtr<SeekableStream> stream)
// {
// auto io_context = TRY(Media::FFmpeg::FFmpegIOContext::create(stream));
// auto loader = make<FFmpegLoaderPlugin>(move(stream), move(io_context));
// TRY(loader->initialize());
// return loader;
// }

ErrorOr<void> FFmpegLoaderPlugin::initialize()
{
Expand Down Expand Up @@ -123,17 +123,17 @@ double FFmpegLoaderPlugin::time_base() const
return av_q2d(m_audio_stream->time_base);
}

bool FFmpegLoaderPlugin::sniff(SeekableStream& stream)
{
auto io_context = MUST(Media::FFmpeg::FFmpegIOContext::create(stream));
#ifdef USE_CONSTIFIED_POINTERS
AVInputFormat const* detected_format {};
#else
AVInputFormat* detected_format {};
#endif
auto score = av_probe_input_buffer2(io_context->avio_context(), &detected_format, nullptr, nullptr, 0, BUFFER_MAX_PROBE_SIZE);
return score > 0;
}
// bool FFmpegLoaderPlugin::sniff(SeekableStream& stream)
// {
// auto io_context = MUST(Media::FFmpeg::FFmpegIOContext::create(stream));
// #ifdef USE_CONSTIFIED_POINTERS
// AVInputFormat const* detected_format {};
// #else
// AVInputFormat* detected_format {};
// #endif
// auto score = av_probe_input_buffer2(io_context->avio_context(), &detected_format, nullptr, nullptr, 0, BUFFER_MAX_PROBE_SIZE);
// return score > 0;
// }

static ErrorOr<FixedArray<Sample>> extract_samples_from_frame(AVFrame& frame)
{
Expand Down
4 changes: 2 additions & 2 deletions Libraries/LibMedia/Audio/FFmpegLoader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ class FFmpegLoaderPlugin : public LoaderPlugin {
explicit FFmpegLoaderPlugin(NonnullOwnPtr<SeekableStream>, NonnullOwnPtr<Media::FFmpeg::FFmpegIOContext>);
virtual ~FFmpegLoaderPlugin();

static bool sniff(SeekableStream& stream);
static ErrorOr<NonnullOwnPtr<LoaderPlugin>> create(NonnullOwnPtr<SeekableStream>);
// static bool sniff(SeekableStream& stream);
// static ErrorOr<NonnullOwnPtr<LoaderPlugin>> create(NonnullOwnPtr<SeekableStream>);

virtual ErrorOr<Vector<FixedArray<Sample>>> load_chunks(size_t samples_to_read_from_input) override;

Expand Down
2 changes: 1 addition & 1 deletion Libraries/LibMedia/Audio/Loader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ struct LoaderPluginInitializer {
};

static constexpr LoaderPluginInitializer s_initializers[] = {
{ FFmpegLoaderPlugin::sniff, FFmpegLoaderPlugin::create },
// { FFmpegLoaderPlugin::sniff, FFmpegLoaderPlugin::create },
};

ErrorOr<NonnullRefPtr<Loader>> Loader::create(StringView path)
Expand Down
2 changes: 2 additions & 0 deletions Libraries/LibMedia/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ set(SOURCES
Color/TransferCharacteristics.cpp
Containers/Matroska/MatroskaDemuxer.cpp
Containers/Matroska/Reader.cpp
IncrementallyPopulatedStream.cpp
PlaybackManager.cpp
PlaybackStates/PlaybackStateHandler.cpp
PlaybackStates/PlayingStateHandler.cpp
Providers/AudioDataProvider.cpp
Providers/GenericTimeProvider.cpp
Providers/VideoDataProvider.cpp
Expand Down
8 changes: 4 additions & 4 deletions Libraries/LibMedia/Containers/Matroska/Document.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,10 +183,10 @@ class Block {
bool discardable() const { return m_discardable; }
void set_discardable(bool discardable) { m_discardable = discardable; }

void set_frames(Vector<ReadonlyBytes>&& frames) { m_frames = move(frames); }
ReadonlyBytes const& frame(size_t index) const { return frames()[index]; }
void set_frames(Vector<ByteBuffer>&& frames) { m_frames = move(frames); }
ByteBuffer const& frame(size_t index) const { return frames()[index]; }
u64 frame_count() const { return m_frames.size(); }
Vector<ReadonlyBytes> const& frames() const { return m_frames; }
Vector<ByteBuffer> const& frames() const { return m_frames; }

private:
u64 m_track_number { 0 };
Expand All @@ -195,7 +195,7 @@ class Block {
bool m_invisible { false };
Lacing m_lacing { None };
bool m_discardable { true };
Vector<ReadonlyBytes> m_frames;
Vector<ByteBuffer> m_frames;
};

class Cluster {
Expand Down
19 changes: 7 additions & 12 deletions Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,9 @@

namespace Media::Matroska {

DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> MatroskaDemuxer::from_file(StringView filename)
DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> MatroskaDemuxer::from_stream(NonnullRefPtr<IncrementallyPopulatedStream> const& stream)
{
return make_ref_counted<MatroskaDemuxer>(TRY(Reader::from_file(filename)));
}

DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> MatroskaDemuxer::from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file)
{
return make_ref_counted<MatroskaDemuxer>(TRY(Reader::from_mapped_file(move(mapped_file))));
}

DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> MatroskaDemuxer::from_data(ReadonlyBytes data)
{
return make_ref_counted<MatroskaDemuxer>(TRY(Reader::from_data(data)));
return make_ref_counted<MatroskaDemuxer>(TRY(Reader::from_stream(stream)));
}

static TrackEntry::TrackType matroska_track_type_from_track_type(TrackType type)
Expand Down Expand Up @@ -200,4 +190,9 @@ DecoderErrorOr<AK::Duration> MatroskaDemuxer::duration_of_track(Track const&)
return total_duration();
}

void MatroskaDemuxer::cancel_blocking_reads()
{
m_reader.abort_blocking_reads();
}

}
10 changes: 4 additions & 6 deletions Libraries/LibMedia/Containers/Matroska/MatroskaDemuxer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,15 @@
#include <AK/HashMap.h>
#include <LibMedia/Demuxer.h>
#include <LibMedia/Export.h>
#include <LibMedia/IncrementallyPopulatedStream.h>

#include "Reader.h"

namespace Media::Matroska {

class MEDIA_API MatroskaDemuxer final : public Demuxer {
public:
// FIXME: We should instead accept some abstract data streaming type so that the demuxer
// can work with non-contiguous data.
static DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> from_file(StringView filename);
static DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> from_mapped_file(NonnullOwnPtr<Core::MappedFile> mapped_file);

static DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> from_data(ReadonlyBytes data);
static DecoderErrorOr<NonnullRefPtr<MatroskaDemuxer>> from_stream(NonnullRefPtr<IncrementallyPopulatedStream> const&);

MatroskaDemuxer(Reader&& reader)
: m_reader(move(reader))
Expand All @@ -42,6 +38,8 @@ class MEDIA_API MatroskaDemuxer final : public Demuxer {

DecoderErrorOr<CodedFrame> get_next_sample_for_track(Track const&) override;

void cancel_blocking_reads() override;

private:
struct TrackStatus {
SampleIterator iterator;
Expand Down
Loading
Loading