From 8993731973e367c23d7714839e6492c54bad5194 Mon Sep 17 00:00:00 2001 From: sa_ddam213 Date: Wed, 22 May 2024 12:03:23 +1200 Subject: [PATCH] Remove FFMpegCore dependency --- OnnxStack.Core/Image/OnnxImage.cs | 5 +- OnnxStack.Core/OnnxStack.Core.csproj | 3 +- OnnxStack.Core/Video/OnnxVideo.cs | 2 +- OnnxStack.Core/Video/VideoHelper.cs | 67 ++++++++++++++++++++----- OnnxStack.Core/Video/VideoInfo.cs | 75 +++++++++++++++++++++++++++- 5 files changed, 133 insertions(+), 19 deletions(-) diff --git a/OnnxStack.Core/Image/OnnxImage.cs b/OnnxStack.Core/Image/OnnxImage.cs index 6cf0890..1716eeb 100644 --- a/OnnxStack.Core/Image/OnnxImage.cs +++ b/OnnxStack.Core/Image/OnnxImage.cs @@ -5,6 +5,7 @@ using SixLabors.ImageSharp.Processing; using System; using System.IO; +using System.Threading; using System.Threading.Tasks; using ImageSharp = SixLabors.ImageSharp.Image; @@ -209,9 +210,9 @@ public void CopyToStream(Stream destination) /// /// The destination. /// - public Task CopyToStreamAsync(Stream destination) + public Task CopyToStreamAsync(Stream destination, CancellationToken cancellationToken) { - return _imageData.SaveAsPngAsync(destination); + return _imageData.SaveAsPngAsync(destination, cancellationToken); } diff --git a/OnnxStack.Core/OnnxStack.Core.csproj b/OnnxStack.Core/OnnxStack.Core.csproj index 17bfa5a..8e9d484 100644 --- a/OnnxStack.Core/OnnxStack.Core.csproj +++ b/OnnxStack.Core/OnnxStack.Core.csproj @@ -1,7 +1,7 @@  - 0.31.0 + 0.31.10 net7.0 disable disable @@ -37,7 +37,6 @@ - diff --git a/OnnxStack.Core/Video/OnnxVideo.cs b/OnnxStack.Core/Video/OnnxVideo.cs index ee8f3e9..dc6f3f9 100644 --- a/OnnxStack.Core/Video/OnnxVideo.cs +++ b/OnnxStack.Core/Video/OnnxVideo.cs @@ -164,7 +164,7 @@ public void Dispose() public static async Task FromFileAsync(string filename, float? frameRate = default, CancellationToken cancellationToken = default) { var videoBytes = await File.ReadAllBytesAsync(filename, cancellationToken); - var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoBytes); + var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoBytes, cancellationToken); if (frameRate.HasValue) videoInfo = videoInfo with { FrameRate = Math.Min(videoInfo.FrameRate, frameRate.Value) }; diff --git a/OnnxStack.Core/Video/VideoHelper.cs b/OnnxStack.Core/Video/VideoHelper.cs index b8c6721..abf8892 100644 --- a/OnnxStack.Core/Video/VideoHelper.cs +++ b/OnnxStack.Core/Video/VideoHelper.cs @@ -1,5 +1,4 @@ -using FFMpegCore; -using OnnxStack.Core.Config; +using OnnxStack.Core.Config; using OnnxStack.Core.Image; using System; using System.Collections.Generic; @@ -7,6 +6,7 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -73,7 +73,7 @@ private static async Task WriteVideoFramesAsync(IEnumerable onnxImage foreach (var image in onnxImages) { // Write each frame to the input stream of FFMPEG - await videoWriter.StandardInput.BaseStream.WriteAsync(image.GetImageBytes(), cancellationToken); + await image.CopyToStreamAsync(videoWriter.StandardInput.BaseStream, cancellationToken); } // Done close stream and wait for app to process @@ -103,7 +103,7 @@ public static async Task WriteVideoStreamAsync(VideoInfo videoInfo, IAsyncEnumer await foreach (var frame in videoStream) { // Write each frame to the input stream of FFMPEG - await frame.CopyToStreamAsync(videoWriter.StandardInput.BaseStream); + await frame.CopyToStreamAsync(videoWriter.StandardInput.BaseStream, cancellationToken); } // Done close stream and wait for app to process @@ -118,12 +118,17 @@ public static async Task WriteVideoStreamAsync(VideoInfo videoInfo, IAsyncEnumer /// /// The video bytes. /// - public static async Task ReadVideoInfoAsync(byte[] videoBytes) + public static async Task ReadVideoInfoAsync(byte[] videoBytes, CancellationToken cancellationToken = default) { - using (var memoryStream = new MemoryStream(videoBytes)) + string tempVideoPath = GetTempFilename(); + try + { + await File.WriteAllBytesAsync(tempVideoPath, videoBytes, cancellationToken); + return await ReadVideoInfoAsync(tempVideoPath, cancellationToken); + } + finally { - var result = await FFProbe.AnalyseAsync(memoryStream).ConfigureAwait(false); - return new VideoInfo(result.PrimaryVideoStream.Width, result.PrimaryVideoStream.Height, result.Duration, (int)result.PrimaryVideoStream.FrameRate); + DeleteTempFile(tempVideoPath); } } @@ -133,10 +138,29 @@ public static async Task ReadVideoInfoAsync(byte[] videoBytes) /// /// The filename. /// - public static async Task ReadVideoInfoAsync(string filename) + public static async Task ReadVideoInfoAsync(string filename, CancellationToken cancellationToken = default) { - var result = await FFProbe.AnalyseAsync(filename).ConfigureAwait(false); - return new VideoInfo(result.PrimaryVideoStream.Width, result.PrimaryVideoStream.Height, result.Duration, (int)result.PrimaryVideoStream.FrameRate); + + using (var metadataReader = CreateMetadataReader(filename)) + { + // Start FFMPEG + metadataReader.Start(); + + var videoInfo = default(VideoInfo); + using (StreamReader reader = metadataReader.StandardOutput) + { + string result = await reader.ReadToEndAsync(); + var videoMetadata = JsonSerializer.Deserialize(result); + var videoStream = videoMetadata.Streams.FirstOrDefault(); + if (videoStream is null) + throw new Exception("Failed to parse video stream metadata"); + + videoInfo = new VideoInfo(videoStream.Height, videoStream.Width, videoStream.Duration, videoStream.FramesPerSecond); + } + + await metadataReader.WaitForExitAsync(cancellationToken); + return videoInfo; + } } @@ -308,7 +332,7 @@ private static Process CreateReader(string inputFile, float fps) { var ffmpegProcess = new Process(); ffmpegProcess.StartInfo.FileName = _configuration.FFmpegPath; - ffmpegProcess.StartInfo.Arguments = $"-hide_banner -loglevel error -i \"{inputFile}\" -c:v png -r {fps} -f image2pipe -"; + ffmpegProcess.StartInfo.Arguments = $"-hide_banner -loglevel error -hwaccel:v auto -i \"{inputFile}\" -c:v png -r {fps} -f image2pipe -"; ffmpegProcess.StartInfo.RedirectStandardOutput = true; ffmpegProcess.StartInfo.UseShellExecute = false; ffmpegProcess.StartInfo.CreateNoWindow = true; @@ -329,7 +353,7 @@ private static Process CreateWriter(string outputFile, float fps, double aspectR var codec = preserveTransparency ? "png" : "libx264"; var format = preserveTransparency ? "yuva420p" : "yuv420p"; ffmpegProcess.StartInfo.FileName = _configuration.FFmpegPath; - ffmpegProcess.StartInfo.Arguments = $"-hide_banner -loglevel error -framerate {fps:F4} -i - -c:v {codec} -movflags +faststart -vf format={format} -aspect {aspectRatio} {outputFile}"; + ffmpegProcess.StartInfo.Arguments = $"-hide_banner -loglevel error -framerate {fps:F4} -hwaccel:v auto -i - -c:v {codec} -movflags +faststart -vf format={format} -aspect {aspectRatio} {outputFile}"; ffmpegProcess.StartInfo.RedirectStandardInput = true; ffmpegProcess.StartInfo.UseShellExecute = false; ffmpegProcess.StartInfo.CreateNoWindow = true; @@ -337,6 +361,23 @@ private static Process CreateWriter(string outputFile, float fps, double aspectR } + /// + /// Creates the metadata reader. + /// + /// The input file. + /// + private static Process CreateMetadataReader(string inputFile) + { + var ffprobeProcess = new Process(); + ffprobeProcess.StartInfo.FileName = _configuration.FFprobePath; + ffprobeProcess.StartInfo.Arguments = $"-v quiet -print_format json -show_format -show_streams {inputFile}"; + ffprobeProcess.StartInfo.RedirectStandardOutput = true; + ffprobeProcess.StartInfo.UseShellExecute = false; + ffprobeProcess.StartInfo.CreateNoWindow = true; + return ffprobeProcess; + } + + /// /// Determines whether we are at the start of a PNG image in the specified buffer. /// diff --git a/OnnxStack.Core/Video/VideoInfo.cs b/OnnxStack.Core/Video/VideoInfo.cs index 7b16855..fb6b626 100644 --- a/OnnxStack.Core/Video/VideoInfo.cs +++ b/OnnxStack.Core/Video/VideoInfo.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; namespace OnnxStack.Core.Video { @@ -12,6 +14,77 @@ public VideoInfo(int height, int width, TimeSpan duration, float frameRate) : th public int Height { get; set; } public int Width { get; set; } - public double AspectRatio => (double)Height / Width; + public double AspectRatio => (double)Width / Height; + } + + public record VideoMetadata + { + [JsonPropertyName("format")] + public VideoFormat Format { get; set; } + + [JsonPropertyName("streams")] + public List Streams { get; set; } + } + + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public record VideoFormat + { + [JsonPropertyName("filename")] + public string FileName { get; set; } + + [JsonPropertyName("nb_streams")] + public int StreamCount { get; set; } + + [JsonPropertyName("format_name")] + public string FormatName { get; set; } + + [JsonPropertyName("format_long_name")] + public string FormatLongName { get; set; } + + [JsonPropertyName("size")] + public long Size { get; set; } + + [JsonPropertyName("bit_rate")] + public long BitRate { get; set; } + } + + [JsonNumberHandling(JsonNumberHandling.AllowReadingFromString)] + public record VideoStream + { + [JsonPropertyName("codec_type")] + public string Type { get; set; } + + [JsonPropertyName("codec_name")] + public string CodecName { get; set; } + + [JsonPropertyName("codec_long_name")] + public string CodecLongName { get; set; } + + [JsonPropertyName("pix_fmt")] + public string PixelFormat { get; set; } + + [JsonPropertyName("width")] + public int Width { get; set; } + + [JsonPropertyName("height")] + public int Height { get; set; } + + [JsonPropertyName("nb_frames")] + public int FrameCount { get; set; } + + [JsonPropertyName("duration")] + public float DurationSeconds { get; set; } + + public float FramesPerSecond => GetFramesPerSecond(); + + public TimeSpan Duration => TimeSpan.FromSeconds(DurationSeconds); + + private float GetFramesPerSecond() + { + if (FrameCount == 0 || DurationSeconds == 0) + return 0; + + return FrameCount / DurationSeconds; + } } }