-
-
Notifications
You must be signed in to change notification settings - Fork 22
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1005 from b-editor/avfoundation
Implement MediaReader/Writer Classes Using AVFoundation
- Loading branch information
Showing
20 changed files
with
1,701 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| using System.Diagnostics; | ||
| using Beutl.Media; | ||
| using Beutl.Media.Pixel; | ||
| using MonoMac.CoreGraphics; | ||
| using MonoMac.CoreImage; | ||
| using MonoMac.CoreMedia; | ||
| using MonoMac.CoreVideo; | ||
| using MonoMac.Foundation; | ||
|
|
||
| namespace Beutl.Extensions.AVFoundation; | ||
|
|
||
| public class AVFSampleUtilities | ||
| { | ||
| public static unsafe CVPixelBuffer? ConvertToCVPixelBuffer(Bitmap<Bgra8888> bitmap) | ||
| { | ||
| int width = bitmap.Width; | ||
| int height = bitmap.Height; | ||
| var pixelBuffer = new CVPixelBuffer(width, height, CVPixelFormatType.CV32BGRA, new CVPixelBufferAttributes | ||
| { | ||
| PixelFormatType = CVPixelFormatType.CV32BGRA, | ||
| Width = width, | ||
| Height = height | ||
| }); | ||
|
|
||
| var r = pixelBuffer.Lock(CVOptionFlags.None); | ||
| if (r != CVReturn.Success) return null; | ||
|
|
||
| Buffer.MemoryCopy( | ||
| (void*)bitmap.Data, (void*)pixelBuffer.GetBaseAddress(0), | ||
| bitmap.ByteCount, bitmap.ByteCount); | ||
|
|
||
| pixelBuffer.Unlock(CVOptionFlags.None); | ||
|
|
||
| return pixelBuffer; | ||
| } | ||
|
|
||
| public static unsafe Bitmap<Bgra8888>? ConvertToBgra(CMSampleBuffer buffer) | ||
| { | ||
| using var imageBuffer = buffer.GetImageBuffer(); | ||
| if (imageBuffer is not CVPixelBuffer pixelBuffer) return null; | ||
|
|
||
| var r = pixelBuffer.Lock(CVOptionFlags.None); | ||
| if (r != CVReturn.Success) return null; | ||
|
|
||
| int width = pixelBuffer.Width; | ||
| int height = pixelBuffer.Height; | ||
| var bitmap = new Bitmap<Bgra8888>(width, height); | ||
| if (pixelBuffer.ColorSpace.Model == CGColorSpaceModel.RGB && pixelBuffer.BytesPerRow == width * 4) | ||
| { | ||
| Buffer.MemoryCopy( | ||
| (void*)pixelBuffer.GetBaseAddress(0), (void*)bitmap.Data, | ||
| bitmap.ByteCount, bitmap.ByteCount); | ||
| pixelBuffer.Unlock(CVOptionFlags.None); | ||
| Parallel.For(0, width * height, i => | ||
| { | ||
| // argb | ||
| // bgra | ||
| var o = bitmap.DataSpan[i]; | ||
| bitmap.DataSpan[i] = new Bgra8888(o.G, o.R, o.A, o.B); | ||
| }); | ||
| return bitmap; | ||
| } | ||
|
|
||
| int bytesPerRow = width * height * 4; | ||
| using (CGColorSpace colorSpace = CGColorSpace.CreateDeviceRGB()) | ||
| using (var cgContext = new CGBitmapContext( | ||
| bitmap.Data, width, height, | ||
| 8, bytesPerRow, colorSpace, | ||
| CGBitmapFlags.ByteOrderDefault | CGBitmapFlags.PremultipliedFirst)) | ||
| using (var ciImage = CIImage.FromImageBuffer(imageBuffer)) | ||
| using (var ciContext = new CIContext(NSObjectFlag.Empty)) | ||
| // CreateCGImageで落ちる、例外なしに | ||
| using (var cgImage = ciContext.CreateCGImage( | ||
| ciImage, new CGRect(0, 0, width, height), (long)CIFormat.ARGB8, colorSpace)) | ||
| { | ||
| cgContext.DrawImage(new CGRect(0, 0, width, height), cgImage); | ||
| } | ||
|
|
||
| pixelBuffer.Unlock(CVOptionFlags.None); | ||
|
|
||
| return bitmap; | ||
| } | ||
|
|
||
| public static int SampleCopyToBuffer(CMSampleBuffer buffer, nint buf, int copyBufferPos, | ||
| int copyBufferSize) | ||
| { | ||
| using var dataBuffer = buffer.GetDataBuffer(); | ||
| Debug.Assert((copyBufferPos + copyBufferSize) <= dataBuffer.DataLength); | ||
| dataBuffer.CopyDataBytes((uint)copyBufferPos, (uint)copyBufferSize, buf); | ||
|
|
||
| return copyBufferSize; | ||
| } | ||
|
|
||
| public static int SampleCopyToBuffer(CMSampleBuffer buffer, nint buf, int copyBufferSize) | ||
| { | ||
| return SampleCopyToBuffer(buffer, buf, 0, copyBufferSize); | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
src/Beutl.Extensions.AVFoundation/Beutl.Extensions.AVFoundation.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <Optimize>false</Optimize> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="MonoMac.NetStandard" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Beutl.Extensibility\Beutl.Extensibility.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| using MonoMac.CoreMedia; | ||
|
|
||
| namespace Beutl.Extensions.AVFoundation.Decoding; | ||
|
|
||
| internal static class CMTimeUtilities | ||
| { | ||
| public static int ConvertFrameFromTimeStamp(CMTime timestamp, double rate) | ||
| { | ||
| return (int)Math.Round(timestamp.Seconds * rate, MidpointRounding.AwayFromZero); | ||
| } | ||
|
|
||
| public static CMTime ConvertTimeStampFromFrame(int frame, double rate) | ||
| { | ||
| return CMTime.FromSeconds(frame / rate, 1); | ||
| } | ||
| } |
127 changes: 127 additions & 0 deletions
127
src/Beutl.Extensions.AVFoundation/Decoding/AVFAudioSampleCache.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| using Beutl.Collections; | ||
| using Beutl.Logging; | ||
| using Microsoft.Extensions.Logging; | ||
| using MonoMac.CoreMedia; | ||
|
|
||
| namespace Beutl.Extensions.AVFoundation.Decoding; | ||
|
|
||
| public class AVFAudioSampleCache(AVFSampleCacheOptions options) | ||
| { | ||
| private readonly ILogger _logger = Log.CreateLogger<AVFAudioSampleCache>(); | ||
|
|
||
| public const int AudioSampleWaringGapCount = 1000; | ||
|
|
||
| private CircularBuffer<AudioCache> _audioCircularBuffer = new(options.MaxAudioBufferSize); | ||
| private short _nBlockAlign; | ||
|
|
||
| private readonly record struct AudioCache(int StartSampleNum, CMSampleBuffer Sample, int AudioSampleCount) | ||
| { | ||
| public bool CopyBuffer(ref int startSample, ref int copySampleLength, ref nint buffer, short nBlockAlign) | ||
| { | ||
| int querySampleEndPos = startSample + copySampleLength; | ||
| int cacheSampleEndPos = StartSampleNum + AudioSampleCount; | ||
| // キャッシュ内に startSample位置があるかどうか | ||
| if (StartSampleNum <= startSample && startSample < cacheSampleEndPos) | ||
| { | ||
| // 要求サイズがキャッシュを超えるかどうか | ||
| if (querySampleEndPos <= cacheSampleEndPos) | ||
| { | ||
| // キャッシュ内に収まる | ||
| int actualBufferPos = (startSample - StartSampleNum) * nBlockAlign; | ||
| int actualBufferSize = copySampleLength * nBlockAlign; | ||
| AVFSampleUtilities.SampleCopyToBuffer(Sample, buffer, actualBufferPos, actualBufferSize); | ||
|
|
||
| startSample += copySampleLength; | ||
| copySampleLength = 0; | ||
| buffer += actualBufferSize; | ||
|
|
||
| return true; | ||
| } | ||
| else | ||
| { | ||
| // 現在のキャッシュ内のデータをコピーする | ||
| int actualBufferPos = (startSample - StartSampleNum) * nBlockAlign; | ||
| int leftSampleCount = cacheSampleEndPos - startSample; | ||
| int actualleftBufferSize = leftSampleCount * nBlockAlign; | ||
| AVFSampleUtilities.SampleCopyToBuffer(Sample, buffer, actualBufferPos, actualleftBufferSize); | ||
|
|
||
| startSample += leftSampleCount; | ||
| copySampleLength -= leftSampleCount; | ||
| buffer += actualleftBufferSize; | ||
|
|
||
| return true; | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
| } | ||
|
|
||
| public void Reset(short nBlockAlign) | ||
| { | ||
| _nBlockAlign = nBlockAlign; | ||
| CircularBuffer<AudioCache> old = _audioCircularBuffer; | ||
| _audioCircularBuffer = new CircularBuffer<AudioCache>(options.MaxAudioBufferSize); | ||
| foreach (AudioCache item in old) | ||
| { | ||
| item.Sample.Dispose(); | ||
| } | ||
| } | ||
|
|
||
| public void Add(int startSample, CMSampleBuffer buffer) | ||
| { | ||
| int lastAudioSampleNum = LastAudioSampleNumber(); | ||
| if (lastAudioSampleNum != -1) | ||
| { | ||
| int actualAudioSampleNum = lastAudioSampleNum + _audioCircularBuffer.Back().AudioSampleCount; | ||
| if (Math.Abs(startSample - actualAudioSampleNum) > AudioSampleWaringGapCount) | ||
| { | ||
| _logger.LogWarning( | ||
| "sample laggin - lag: {lag} startSample: {startSample} lastAudioSampleNum: {lastAudioSampleNum}", | ||
| startSample - actualAudioSampleNum, | ||
| startSample, | ||
| actualAudioSampleNum); | ||
| } | ||
|
|
||
| startSample = lastAudioSampleNum + _audioCircularBuffer.Back().AudioSampleCount; | ||
| } | ||
|
|
||
| int audioSampleCount = buffer.NumSamples; | ||
|
|
||
| if (_audioCircularBuffer.IsFull) | ||
| { | ||
| _audioCircularBuffer.Front().Sample.Dispose(); | ||
| _audioCircularBuffer.PopFront(); | ||
| } | ||
|
|
||
| var audioCache = new AudioCache(startSample, buffer, audioSampleCount); | ||
| _audioCircularBuffer.PushBack(audioCache); | ||
| } | ||
|
|
||
| public int LastAudioSampleNumber() | ||
| { | ||
| if (_audioCircularBuffer.Size > 0) | ||
| { | ||
| AudioCache prevAudioCache = _audioCircularBuffer.Back(); | ||
| return prevAudioCache.StartSampleNum; | ||
| } | ||
|
|
||
| return -1; | ||
| } | ||
|
|
||
| public bool SearchAudioSampleAndCopyBuffer(int startSample, int copySampleLength, nint buffer) | ||
| { | ||
| foreach (AudioCache audioCache in _audioCircularBuffer) | ||
| { | ||
| if (audioCache.CopyBuffer(ref startSample, ref copySampleLength, ref buffer, _nBlockAlign)) | ||
| { | ||
| if (copySampleLength == 0) | ||
| { | ||
| return true; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
| } |
Oops, something went wrong.