Skip to content

Commit 55c765d

Browse files
ladvocladvoc
authored
Fix video source memory handling (#122)
* Remove extraneous allocations * Fix memory leak
1 parent dc461e8 commit 55c765d

5 files changed

Lines changed: 101 additions & 78 deletions

File tree

‎Runtime/Scripts/CameraVideoSource.cs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public CameraVideoSource(Camera camera, VideoBufferType bufferType = VideoBuff
4141

4242
~CameraVideoSource()
4343
{
44-
Dispose();
44+
Dispose(false);
4545
ClearRenderTexture();
4646
}
4747

@@ -77,27 +77,27 @@ protected override bool ReadBuffer()
7777
_bufferType = GetVideoBufferType(_textureFormat);
7878
_renderTexture = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
7979
Camera.targetTexture = _renderTexture as RenderTexture;
80-
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
81-
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
80+
_captureBuffer = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
81+
_previewTexture = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
8282
textureChanged = true;
8383
}
8484
ScreenCapture.CaptureScreenshotIntoRenderTexture(_renderTexture);
8585

8686
#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
8787
// Flip the texture for OSX
88-
Graphics.CopyTexture(_renderTexture, _dest);
89-
var pixels = _dest.GetPixels();
88+
Graphics.CopyTexture(_renderTexture, _previewTexture);
89+
var pixels = _previewTexture.GetPixels();
9090
var flippedPixels = new Color[pixels.Length];
91-
for (int i = 0; i < _dest.height; i++)
91+
for (int i = 0; i < _previewTexture.height; i++)
9292
{
93-
Array.Copy(pixels, i * _dest.width, flippedPixels, (_dest.height - i - 1) * _dest.width, _dest.width);
93+
Array.Copy(pixels, i * _previewTexture.width, flippedPixels, (_previewTexture.height - i - 1) * _previewTexture.width, _previewTexture.width);
9494
}
95-
_dest.SetPixels(flippedPixels);
95+
_previewTexture.SetPixels(flippedPixels);
9696
#else
97-
Graphics.CopyTexture(_renderTexture, _dest);
97+
Graphics.CopyTexture(_renderTexture, _previewTexture);
9898
#endif
9999

100-
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _renderTexture, 0, _textureFormat, OnReadback);
100+
AsyncGPUReadback.RequestIntoNativeArray(ref _captureBuffer, _renderTexture, 0, _textureFormat, OnReadback);
101101
}
102102
catch (Exception e)
103103
{

‎Runtime/Scripts/RtcVideoSource.cs‎

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@
77
using Unity.Collections.LowLevel.Unsafe;
88
using LiveKit.Internal.FFIClients.Requests;
99
using System.Collections;
10-
using UnityEngine.Experimental.Rendering;
1110

1211
namespace LiveKit
1312
{
14-
public abstract class RtcVideoSource : IRtcSource
13+
public abstract class RtcVideoSource : IRtcSource, IDisposable
1514
{
1615
public enum VideoStreamSource
1716
{
@@ -20,7 +19,6 @@ public enum VideoStreamSource
2019
Camera = 2
2120
}
2221

23-
2422
internal FfiHandle Handle { get; set; }
2523

2624
public abstract int GetWidth();
@@ -32,21 +30,20 @@ public enum VideoStreamSource
3230
/// Called when we receive a new texture (first texture or the resolution changed)
3331
public event TextureReceiveDelegate TextureReceived;
3432

35-
protected Texture2D _dest;
36-
protected NativeArray<byte> _data;
33+
protected Texture2D _previewTexture;
34+
protected NativeArray<byte> _captureBuffer;
3735
protected VideoStreamSource _sourceType;
3836
protected VideoBufferType _bufferType;
3937
protected VideoSourceInfo _info;
4038
protected bool _reading = false;
4139
protected bool _requestPending = false;
42-
protected bool isDisposed = true;
40+
protected bool _disposed = false;
4341
protected bool _playing = false;
4442
private bool _muted = false;
4543
public override bool Muted => _muted;
4644

4745
internal RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType)
4846
{
49-
isDisposed = false;
5047
_sourceType = sourceType;
5148
_bufferType = bufferType;
5249
Handle = null;
@@ -126,25 +123,21 @@ public virtual void Start()
126123

127124
public virtual void Stop()
128125
{
129-
_playing = false;
126+
_playing = false;
130127
}
131128

132129
public IEnumerator Update()
133130
{
134131
while (_playing)
135132
{
136133
yield return null;
134+
if (_disposed) break;
137135
var textureChanged = ReadBuffer();
138136

139-
if(textureChanged)
140-
{
141-
TextureReceived?.Invoke(_dest);
142-
}
137+
if (textureChanged)
138+
TextureReceived?.Invoke(_previewTexture);
143139

144-
if(_muted)
145-
{
146-
continue;
147-
}
140+
if (_muted)continue;
148141
SendFrame();
149142
}
150143

@@ -156,26 +149,17 @@ public override void SetMute(bool muted)
156149
_muted = muted;
157150
}
158151

159-
public virtual void Dispose()
160-
{
161-
if (!isDisposed)
162-
{
163-
if (_dest != null) UnityEngine.Object.Destroy(_dest);
164-
isDisposed = true;
165-
}
166-
}
167-
168152
protected abstract bool ReadBuffer();
169153

170154
protected virtual bool SendFrame()
171155
{
172-
var result = _requestPending && !isDisposed;
156+
var result = _requestPending && !_disposed;
173157
if (result)
174158
{
175159
var buffer = new VideoBufferInfo();
176160
unsafe
177161
{
178-
buffer.DataPtr = (ulong)NativeArrayUnsafeUtility.GetUnsafePtr(_data);
162+
buffer.DataPtr = (ulong)NativeArrayUnsafeUtility.GetUnsafePtr(_captureBuffer);
179163
}
180164

181165
buffer.Type = _bufferType;
@@ -210,6 +194,30 @@ protected void OnReadback(AsyncGPUReadbackRequest req)
210194
_reading = false;
211195
}
212196
}
197+
198+
public void Dispose()
199+
{
200+
Dispose(true);
201+
GC.SuppressFinalize(this);
202+
}
203+
204+
protected virtual void Dispose(bool disposing)
205+
{
206+
if (_disposed) return;
207+
if (_previewTexture != null)
208+
UnityEngine.Object.Destroy(_previewTexture);
209+
if (_captureBuffer.IsCreated)
210+
{
211+
Debug.Log("Disposing capture buffer");
212+
_captureBuffer.Dispose();
213+
}
214+
_disposed = true;
215+
}
216+
217+
~RtcVideoSource()
218+
{
219+
Dispose(false);
220+
}
213221
}
214222
}
215223

‎Runtime/Scripts/ScreenVideoSource.cs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public override void Stop()
4141

4242
~ScreenVideoSource()
4343
{
44-
Dispose();
44+
Dispose(false);
4545
ClearRenderTexture();
4646
}
4747

@@ -70,27 +70,27 @@ protected override bool ReadBuffer()
7070
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
7171
_bufferType = GetVideoBufferType(_textureFormat);
7272
_renderTexture = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
73-
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
74-
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
73+
_captureBuffer = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
74+
_previewTexture = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
7575
textureChanged = true;
7676
}
7777
ScreenCapture.CaptureScreenshotIntoRenderTexture(_renderTexture);
7878

7979
#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
8080
// Flip the texture for OSX
81-
Graphics.CopyTexture(_renderTexture, _dest);
82-
var pixels = _dest.GetPixels();
81+
Graphics.CopyTexture(_renderTexture, _previewTexture);
82+
var pixels = _previewTexture.GetPixels();
8383
var flippedPixels = new Color[pixels.Length];
84-
for (int i = 0; i < _dest.height; i++)
84+
for (int i = 0; i < _previewTexture.height; i++)
8585
{
86-
Array.Copy(pixels, i * _dest.width, flippedPixels, (_dest.height - i - 1) * _dest.width, _dest.width);
86+
Array.Copy(pixels, i * _previewTexture.width, flippedPixels, (_previewTexture.height - i - 1) * _previewTexture.width, _previewTexture.width);
8787
}
88-
_dest.SetPixels(flippedPixels);
88+
_previewTexture.SetPixels(flippedPixels);
8989
#else
90-
Graphics.CopyTexture(_renderTexture, _dest);
90+
Graphics.CopyTexture(_renderTexture, _previewTexture);
9191
#endif
9292

93-
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _renderTexture, 0, _textureFormat, OnReadback);
93+
AsyncGPUReadback.RequestIntoNativeArray(ref _captureBuffer, _renderTexture, 0, _textureFormat, OnReadback);
9494
}
9595
catch (Exception e)
9696
{

‎Runtime/Scripts/TextureVideoSource.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBuf
3636

3737
~TextureVideoSource()
3838
{
39-
Dispose();
39+
Dispose(false);
4040
}
4141

4242
// Read the texture data into a native array asynchronously
@@ -47,16 +47,16 @@ protected override bool ReadBuffer()
4747
_reading = true;
4848
var textureChanged = false;
4949

50-
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) {
50+
if (_previewTexture == null || _previewTexture.width != GetWidth() || _previewTexture.height != GetHeight()) {
5151
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
5252
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
5353
_bufferType = GetVideoBufferType(_textureFormat);
54-
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
55-
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
54+
_captureBuffer = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
55+
_previewTexture = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
5656
textureChanged = true;
5757
}
58-
Graphics.CopyTexture(Texture, _dest);
59-
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
58+
Graphics.CopyTexture(Texture, _previewTexture);
59+
AsyncGPUReadback.RequestIntoNativeArray(ref _captureBuffer, _previewTexture, 0, _textureFormat, OnReadback);
6060
return textureChanged;
6161
}
6262
}

‎Runtime/Scripts/WebCameraSource.cs‎

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
using UnityEngine;
22
using LiveKit.Proto;
3-
using UnityEngine.Rendering;
43
using Unity.Collections;
54
using UnityEngine.Experimental.Rendering;
6-
using System;
75
using System.Runtime.InteropServices;
86

97
namespace LiveKit
@@ -14,21 +12,21 @@ public class WebCameraSource : RtcVideoSource
1412
TextureFormat _textureFormat;
1513
private RenderTexture _tempTexture;
1614

17-
public WebCamTexture Texture { get; }
15+
public WebCamTexture CamTexture { get; }
1816

1917
public override int GetWidth()
2018
{
21-
return Texture.width;
19+
return CamTexture.width;
2220
}
2321

2422
public override int GetHeight()
2523
{
26-
return Texture.height;
24+
return CamTexture.height;
2725
}
2826

2927
protected override VideoRotation GetVideoRotation()
3028
{
31-
switch (Texture.videoRotationAngle)
29+
switch (CamTexture.videoRotationAngle)
3230
{
3331
case 90: return VideoRotation._90;
3432
case 180: return VideoRotation._0;
@@ -38,52 +36,69 @@ protected override VideoRotation GetVideoRotation()
3836

3937
public WebCameraSource(WebCamTexture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType)
4038
{
41-
Texture = texture;
39+
CamTexture = texture;
4240
base.Init();
4341
}
4442

4543
~WebCameraSource()
4644
{
47-
Dispose();
45+
Dispose(false);
4846
}
4947

48+
private Color32[] _readBuffer;
49+
5050
// Read the texture data into a native array asynchronously
5151
protected override bool ReadBuffer()
5252
{
53-
if (_reading && !Texture.isPlaying)
53+
if (_reading && !CamTexture.isPlaying)
5454
return false;
5555
_reading = true;
5656
var textureChanged = false;
5757

58-
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
58+
int width = GetWidth();
59+
int height = GetHeight();
60+
61+
if (_previewTexture == null ||
62+
_previewTexture.width != width ||
63+
_previewTexture.height != height)
5964
{
60-
var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
65+
Debug.Log("Creating new texture");
66+
// Required when using Allocator.Persistent
67+
if (_captureBuffer.IsCreated)
68+
_captureBuffer.Dispose();
69+
70+
var compatibleFormat = SystemInfo.GetCompatibleFormat(CamTexture.graphicsFormat, FormatUsage.ReadPixels);
6171
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
6272
_bufferType = GetVideoBufferType(_textureFormat);
63-
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
64-
_dest = new Texture2D(GetWidth(), GetHeight(), TextureFormat.BGRA32, false);
65-
if (Texture.graphicsFormat != _dest.graphicsFormat) _tempTexture = new RenderTexture(GetWidth(), GetHeight(), 0, _dest.graphicsFormat);
73+
74+
_readBuffer = new Color32[width * height];
75+
_previewTexture = new Texture2D(width, height, TextureFormat.BGRA32, false);
76+
_captureBuffer = new NativeArray<byte>(width * height * GetStrideForBuffer(_bufferType), Allocator.Persistent);
77+
78+
if (CamTexture.graphicsFormat != _previewTexture.graphicsFormat)
79+
_tempTexture = new RenderTexture(width, height, 0, _previewTexture.graphicsFormat);
80+
6681
textureChanged = true;
6782
}
6883

69-
Color32[] pixels = new Color32[GetWidth() * GetHeight()];
70-
Texture.GetPixels32(pixels);
71-
var bytes = MemoryMarshal.Cast<Color32, byte>(pixels);
72-
_data.CopyFrom(bytes.ToArray());
84+
CamTexture.GetPixels32(_readBuffer);
85+
MemoryMarshal.Cast<Color32, byte>(_readBuffer)
86+
.CopyTo(_captureBuffer.AsSpan());
87+
7388
_requestPending = true;
74-
75-
if (Texture.graphicsFormat != _dest.graphicsFormat)
89+
90+
if (CamTexture.graphicsFormat != _previewTexture.graphicsFormat)
7691
{
77-
Graphics.Blit(Texture, _tempTexture);
78-
Graphics.CopyTexture(_tempTexture, _dest);
92+
Graphics.Blit(CamTexture, _tempTexture);
93+
Graphics.CopyTexture(_tempTexture, _previewTexture);
7994
}
8095
else
8196
{
8297
#if UNITY_6000_0_OR_NEWER && UNITY_IOS
83-
_dest.SetPixels(Texture.GetPixels());
84-
_dest.Apply();
98+
_previewTexture.SetPixels(CamTexture.GetPixels());
99+
_previewTexture.Apply();
85100
#else
86-
Graphics.CopyTexture(Texture, _dest);
101+
Graphics.CopyTexture(CamTexture, _previewTexture);
87102
#endif
88103
}
89104

0 commit comments

Comments
 (0)