Skip to content

Commit 9b91d02

Browse files
KSemenenkodtymakhovSanyyazzzHurko-VolodymyrHurko Volodymyr
authored
Develop (#65)
* aws * Added Managed Identity for Azure Storage * Release alpha version * base controller * Add methods from StorageFromFileExt and StorageExt (#46) Co-authored-by: Hurko Volodymyr <hurko@managed-code.com> * prototype * test containsers azure + aws * checks * actions * test containsers * test * rename * #54 * refactoring * tests * tests * tests * fix all tests * naming --------- Co-authored-by: dtymakhov <dtymakhov@gmail.com> Co-authored-by: Alexandr Ivanitskyi <6saaanchik6@gmail.com> Co-authored-by: Hurko Volodymyr <92858780+Hurko-Volodymyr@users.noreply.github.com> Co-authored-by: Hurko Volodymyr <hurko@managed-code.com> Co-authored-by: Eduard <eduard@managed-code.com>
1 parent d84613b commit 9b91d02

File tree

105 files changed

+2476
-1140
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+2476
-1140
lines changed

‎.github/workflows/dotnet.yml‎

Lines changed: 19 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -14,75 +14,39 @@ jobs:
1414

1515
build-and-test:
1616
runs-on: ubuntu-latest
17-
services:
18-
localstack:
19-
image: localstack/localstack:latest
20-
ports:
21-
- 4563-4599:4563-4599
22-
- 8055:8080
23-
env:
24-
SERVICES: s3
25-
DEBUG: 1
26-
DATA_DIR: /tmp/localstack/data
27-
AWS_SECRET_KEY: 'localkey'
28-
AWS_BUCKET_NAME: 'managed-code-bucket'
29-
AWS_ACCESS_KEY_ID: 'localkey'
30-
AWS_SECRET_ACCESS_KEY: 'localsecret'
31-
DEFAULT_REGION: 'eu-west-1'
3217

3318
steps:
19+
20+
- name: checkout
21+
uses: actions/checkout@v3
3422

35-
- uses: actions/checkout@v3
3623
- name: Setup .NET
3724
uses: actions/setup-dotnet@v3
3825
with:
3926
dotnet-version: 7.0.x
40-
41-
- name: NDepend
42-
uses: ndepend/ndepend-action@v1
43-
with:
44-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45-
license: ${{ secrets.NDEPENDLICENSE }}
46-
47-
- name: azuright
48-
uses: potatoqualitee/azuright@v1.1
49-
50-
- name: docker run fake-gcs-server
51-
run: |
52-
docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme http -external-url "http://localhost:4443"
53-
sleep 5s
54-
55-
- name: check storage emulators
56-
run: |
57-
curl http://localhost:4443/
58-
curl http://localhost:4566/
59-
curl http://localhost:10000/
60-
61-
# run build and test
27+
6228
- name: Restore dependencies
6329
run: dotnet restore
30+
6431
- name: Build
6532
run: dotnet build --no-restore
33+
6634
- name: Test
6735
run: dotnet test --no-build --logger 'trx;LogFileName=test-results.trx'
68-
env:
69-
DEFAULT_REGION: eu-west-1
70-
AWS_ACCESS_KEY_ID: localkey
71-
AWS_SECRET_ACCESS_KEY: localsecret
36+
7237
- name: Collect Code Coverage
7338
run: dotnet test --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=lcov /p:CoverletOutput=ManagedCode.Storage.Tests/lcov.info
74-
env:
75-
DEFAULT_REGION: eu-west-1
76-
AWS_ACCESS_KEY_ID: localkey
77-
AWS_SECRET_ACCESS_KEY: localsecret
39+
40+
41+
- name: NDepend
42+
uses: ndepend/ndepend-action@v1
43+
with:
44+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45+
license: ${{ secrets.NDEPENDLICENSE }}
46+
coveragefolder: ManagedCode.Storage.Tests
47+
baseline: recent
48+
#baseline: main_recent
7849

79-
# - name: test-reports
80-
# uses: dorny/test-reporter@v1.5.0
81-
# with:
82-
# name: Test Reporter
83-
# reporter: dotnet-trx
84-
# path: ManagedCode.Storage.Tests/test-results.trx
85-
8650
- name : coverlet
8751
uses: b3b00/coverlet-action@1.1.9
8852
with:
@@ -95,3 +59,5 @@ jobs:
9559
with:
9660
github-token: ${{secrets.GITHUB_TOKEN }}
9761
path-to-lcov: ManagedCode.Storage.Tests/lcov.info
62+
63+

‎Directory.Build.props‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
<RepositoryUrl>https://github.com/managedcode/Storage</RepositoryUrl>
1818
<PackageProjectUrl>https://github.com/managedcode/Storage</PackageProjectUrl>
1919
<Product>Managed Code - Storage</Product>
20-
<Version>2.1.14</Version>
21-
<PackageVersion>2.1.14</PackageVersion>
20+
<Version>2.1.15-alpha</Version>
21+
<PackageVersion>2.1.15-alpha</PackageVersion>
2222

2323
</PropertyGroup>
2424
<PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">

‎ManagedCode.Storage.Aws/AWSStorage.cs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public override async Task<Result> RemoveContainerAsync(CancellationToken cancel
3434
}
3535
catch (Exception ex)
3636
{
37-
_logger?.LogError(ex.Message, ex);
37+
_logger.LogException(ex);
3838
return Result.Fail(ex);
3939
}
4040
}
@@ -104,7 +104,7 @@ protected override async Task<Result> CreateContainerInternalAsync(CancellationT
104104
}
105105
catch (Exception ex)
106106
{
107-
_logger?.LogError(ex.Message, ex);
107+
_logger.LogException(ex);
108108
return Result.Fail(ex);
109109
}
110110
}
@@ -124,7 +124,7 @@ protected override async Task<Result> DeleteDirectoryInternalAsync(string direct
124124
}
125125
catch (Exception ex)
126126
{
127-
_logger?.LogError(ex.Message, ex);
127+
_logger.LogException(ex);
128128
return Result.Fail(ex);
129129
}
130130
}
@@ -183,7 +183,7 @@ await localFile.CopyFromStreamAsync(await StorageClient.GetObjectStreamAsync(Sto
183183
}
184184
catch (Exception ex)
185185
{
186-
_logger?.LogError(ex.Message, ex);
186+
_logger.LogException(ex);
187187
return Result<LocalFile>.Fail(ex);
188188
}
189189
}
@@ -211,7 +211,7 @@ await StorageClient.DeleteObjectAsync(new DeleteObjectRequest
211211
}
212212
catch (Exception ex)
213213
{
214-
_logger?.LogError(ex.Message, ex);
214+
_logger.LogException(ex);
215215
return Result<bool>.Fail(ex);
216216
}
217217
}
@@ -235,7 +235,7 @@ protected override async Task<Result<bool>> ExistsInternalAsync(ExistOptions opt
235235
}
236236
catch (Exception ex)
237237
{
238-
_logger?.LogError(ex.Message, ex);
238+
_logger.LogException(ex);
239239
return Result<bool>.Fail(ex);
240240
}
241241
}
@@ -269,7 +269,7 @@ protected override async Task<Result<BlobMetadata>> GetBlobMetadataInternalAsync
269269
}
270270
catch (Exception ex)
271271
{
272-
_logger?.LogError(ex.Message, ex);
272+
_logger.LogException(ex);
273273
return Result<BlobMetadata>.Fail(ex);
274274
}
275275
}
@@ -301,7 +301,7 @@ protected override async Task<Result> SetLegalHoldInternalAsync(bool hasLegalHol
301301
}
302302
catch (Exception ex)
303303
{
304-
_logger?.LogError(ex.Message, ex);
304+
_logger.LogException(ex);
305305
return Result.Fail(ex);
306306
}
307307
}
@@ -323,7 +323,7 @@ protected override async Task<Result<bool>> HasLegalHoldInternalAsync(LegalHoldO
323323
}
324324
catch (Exception ex)
325325
{
326-
_logger?.LogError(ex.Message, ex);
326+
_logger.LogException(ex);
327327
return Result<bool>.Fail(ex);
328328
}
329329
}

‎ManagedCode.Storage.Aws/BlobStream.cs‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
using System;
2+
using System.Collections.Concurrent;
3+
using System.Collections.Generic;
4+
using System.IO;
5+
using System.Linq;
6+
using System.Threading.Tasks;
7+
using Amazon.S3;
8+
using Amazon.S3.Model;
9+
10+
namespace ManagedCode.Storage.Aws;
11+
12+
public class BlobStream : Stream
13+
{
14+
/* Note the that maximum size (as of now) of a file in S3 is 5TB so it isn't
15+
* safe to assume all uploads will work here. MAX_PART_SIZE times MAX_PART_COUNT
16+
* is ~50TB, which is too big for S3. */
17+
const long MIN_PART_LENGTH = 5L * 1024 * 1024; // all parts but the last this size or greater
18+
const long MAX_PART_LENGTH = 5L * 1024 * 1024 * 1024; // 5GB max per PUT
19+
const long MAX_PART_COUNT = 10000; // no more than 10,000 parts total
20+
const long DEFAULT_PART_LENGTH = MIN_PART_LENGTH;
21+
22+
internal class Metadata
23+
{
24+
public string BucketName;
25+
public string Key;
26+
public long PartLength = DEFAULT_PART_LENGTH;
27+
28+
public int PartCount = 0;
29+
public string UploadId;
30+
public MemoryStream CurrentStream;
31+
32+
public long Position = 0; // based on bytes written
33+
public long Length = 0; // based on bytes written or SetLength, whichever is larger (no truncation)
34+
35+
public List<Task> Tasks = new List<Task>();
36+
public ConcurrentDictionary<int, string> PartETags = new ConcurrentDictionary<int, string>();
37+
}
38+
39+
Metadata _metadata = new Metadata();
40+
IAmazonS3 _s3 = null;
41+
42+
public BlobStream(IAmazonS3 s3, string s3uri, long partLength = DEFAULT_PART_LENGTH)
43+
: this(s3, new Uri(s3uri), partLength)
44+
{
45+
}
46+
47+
public BlobStream(IAmazonS3 s3, Uri s3uri, long partLength = DEFAULT_PART_LENGTH)
48+
: this (s3, s3uri.Host, s3uri.LocalPath.Substring(1), partLength)
49+
{
50+
}
51+
52+
public BlobStream(IAmazonS3 s3, string bucket, string key, long partLength = DEFAULT_PART_LENGTH)
53+
{
54+
_s3 = s3;
55+
_metadata.BucketName = bucket;
56+
_metadata.Key = key;
57+
_metadata.PartLength = partLength;
58+
}
59+
60+
protected override void Dispose(bool disposing)
61+
{
62+
if (disposing)
63+
{
64+
if (_metadata != null)
65+
{
66+
Flush(true);
67+
CompleteUpload();
68+
}
69+
}
70+
_metadata = null;
71+
base.Dispose(disposing);
72+
}
73+
74+
public override bool CanRead => false;
75+
public override bool CanSeek => false;
76+
public override bool CanWrite => true;
77+
public override long Length => _metadata.Length = Math.Max(_metadata.Length, _metadata.Position);
78+
79+
public override long Position
80+
{
81+
get => _metadata.Position;
82+
set => throw new NotImplementedException();
83+
}
84+
85+
public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException();
86+
public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException();
87+
88+
public override void SetLength(long value)
89+
{
90+
_metadata.Length = Math.Max(_metadata.Length, value);
91+
_metadata.PartLength = Math.Max(MIN_PART_LENGTH, Math.Min(MAX_PART_LENGTH, _metadata.Length / MAX_PART_COUNT));
92+
}
93+
94+
private void StartNewPart()
95+
{
96+
if (_metadata.CurrentStream != null) {
97+
Flush(false);
98+
}
99+
_metadata.CurrentStream = new MemoryStream();
100+
_metadata.PartLength = Math.Min(MAX_PART_LENGTH, Math.Max(_metadata.PartLength, (_metadata.PartCount / 2 + 1) * MIN_PART_LENGTH));
101+
}
102+
103+
public override void Flush()
104+
{
105+
Flush(false);
106+
}
107+
108+
private void Flush(bool disposing)
109+
{
110+
if ((_metadata.CurrentStream == null || _metadata.CurrentStream.Length < MIN_PART_LENGTH) &&
111+
!disposing)
112+
return;
113+
114+
if (_metadata.UploadId == null) {
115+
_metadata.UploadId = _s3.InitiateMultipartUploadAsync(new InitiateMultipartUploadRequest()
116+
{
117+
BucketName = _metadata.BucketName,
118+
Key = _metadata.Key
119+
}).GetAwaiter().GetResult().UploadId;
120+
}
121+
122+
if (_metadata.CurrentStream != null)
123+
{
124+
var i = ++_metadata.PartCount;
125+
126+
_metadata.CurrentStream.Seek(0, SeekOrigin.Begin);
127+
var request = new UploadPartRequest()
128+
{
129+
BucketName = _metadata.BucketName,
130+
Key = _metadata.Key,
131+
UploadId = _metadata.UploadId,
132+
PartNumber = i,
133+
IsLastPart = disposing,
134+
InputStream = _metadata.CurrentStream
135+
};
136+
_metadata.CurrentStream = null;
137+
138+
var upload = Task.Run(async () =>
139+
{
140+
var response = await _s3.UploadPartAsync(request);
141+
_metadata.PartETags.AddOrUpdate(i, response.ETag,
142+
(n, s) => response.ETag);
143+
request.InputStream.Dispose();
144+
});
145+
_metadata.Tasks.Add(upload);
146+
}
147+
}
148+
149+
private void CompleteUpload()
150+
{
151+
Task.WaitAll(_metadata.Tasks.ToArray());
152+
153+
if (Length > 0) {
154+
_s3.CompleteMultipartUploadAsync(new CompleteMultipartUploadRequest()
155+
{
156+
BucketName = _metadata.BucketName,
157+
Key = _metadata.Key,
158+
PartETags = _metadata.PartETags.Select(e => new PartETag(e.Key, e.Value)).ToList(),
159+
UploadId = _metadata.UploadId
160+
}).GetAwaiter().GetResult();
161+
}
162+
}
163+
164+
public override void Write(byte[] buffer, int offset, int count)
165+
{
166+
if (count == 0) return;
167+
168+
// write as much of the buffer as will fit to the current part, and if needed
169+
// allocate a new part and continue writing to it (and so on).
170+
var o = offset;
171+
var c = Math.Min(count, buffer.Length - offset); // don't over-read the buffer, even if asked to
172+
do
173+
{
174+
if (_metadata.CurrentStream == null || _metadata.CurrentStream.Length >= _metadata.PartLength)
175+
StartNewPart();
176+
177+
var remaining = _metadata.PartLength - _metadata.CurrentStream.Length;
178+
var w = Math.Min(c, (int)remaining);
179+
_metadata.CurrentStream.Write(buffer, o, w);
180+
181+
_metadata.Position += w;
182+
c -= w;
183+
o += w;
184+
} while (c > 0);
185+
}
186+
}

‎ManagedCode.Storage.Aws/ManagedCode.Storage.Aws.csproj‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@
2222
</ItemGroup>
2323

2424
<ItemGroup>
25-
<PackageReference Include="ManagedCode.Communication" Version="2.0.23" />
26-
<PackageReference Include="AWSSDK.S3" Version="3.7.104.19" />
25+
<PackageReference Include="ManagedCode.Communication" Version="2.0.26" />
26+
<PackageReference Include="AWSSDK.S3" Version="3.7.205.10" />
2727
<PackageReference Include="Humanizer.Core" Version="2.14.1" />
28-
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
28+
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
2929
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
3030
</ItemGroup>
3131

0 commit comments

Comments
 (0)