From 94462ac9e54e79c88322af7861cf059dedaeb068 Mon Sep 17 00:00:00 2001 From: Marlon Regenhardt Date: Mon, 25 Mar 2024 12:41:29 +0100 Subject: [PATCH] feat(Storage): Add Azure Blob Storage provider (#128) Closes #127 --- Directory.Packages.props | 1 + docs/docs/Index.md | 2 +- docs/docs/Installation/azure.md | 42 ++++++---- docs/docusaurus.config.ts | 2 +- .../AzureApplicationExtensions.cs | 59 +++++++------- src/BaGetter.Azure/BaGetter.Azure.csproj | 4 +- .../Configuration/AzureBlobStorageOptions.cs | 2 +- .../Extensions/StorageExceptionExtensions.cs | 31 ++++---- .../Storage/BlobStorageService.cs | 76 ++++++++++--------- src/BaGetter/Startup.cs | 2 +- 10 files changed, 117 insertions(+), 104 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index b440c9775..233d1feee 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,6 +5,7 @@ + diff --git a/docs/docs/Index.md b/docs/docs/Index.md index 40844aa94..25b935270 100644 --- a/docs/docs/Index.md +++ b/docs/docs/Index.md @@ -16,7 +16,7 @@ BaGetter (pronounced "ba getter") is a lightweight NuGet and symbol server. It i -BaGetter supports Filesystem, GCP and AWS S3 buckets for package storage, and MySQL, Sqlite, SqlServer and PostgreSQL as database. The current per-package size limit is ~8GB. It can be hosted on IIS, and is also available in a linux [docker image](https://hub.docker.com/r/bagetter/bagetter). +BaGetter supports Filesystem, GCP and AWS S3 buckets, and Azure Blob Storage for package storage, and MySQL, Sqlite, SqlServer and PostgreSQL as database. The current per-package size limit is ~8GB. It can be hosted on IIS, and is also available in a linux [docker image](https://hub.docker.com/r/bagetter/bagetter). ## Run BaGetter diff --git a/docs/docs/Installation/azure.md b/docs/docs/Installation/azure.md index 050b1a15e..1733c96ff 100644 --- a/docs/docs/Installation/azure.md +++ b/docs/docs/Installation/azure.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Run BaGetter on Azure :::warning @@ -6,7 +9,7 @@ This page is a work in progress! ::: -Use Azure to scale BaGetter. You can store metadata on [Azure SQL Database](https://azure.microsoft.com/en-us/services/sql-database/), upload packages to [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/), and provide powerful search using [Azure Search](https://azure.microsoft.com/en-us/services/search/). +Use Azure to scale BaGetter. You can store metadata on [Azure SQL Database](https://azure.microsoft.com/products/azure-sql/database/), upload packages to [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs/), and soon provide powerful search using [Azure Search](https://azure.microsoft.com/en-us/services/search/). ## TODO @@ -16,11 +19,11 @@ Use Azure to scale BaGetter. You can store metadata on [Azure SQL Database](http ## Configure BaGetter -You can modify BaGetter's configurations by editing the `appsettings.json` file. For the full list of configurations, please refer to [BaGetter's configuration](../configuration.md) guide. +You can modify BaGetter's configurations by editing the `appsettings.json` file or through [environment variables](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/?view=aspnetcore-8.0#non-prefixed-environment-variables). For the full list of configurations, please refer to [BaGetter's configuration](../configuration.md) guide. ### Azure SQL database -Update the `appsettings.json` file: +Set the database type to `SqlServer` and provide a [connection string](https://learn.microsoft.com/ef/core/miscellaneous/connection-strings): ```json { @@ -37,24 +40,32 @@ Update the `appsettings.json` file: ### Azure Blob Storage -Update the `appsettings.json` file: +Set the storage type to `AzureBlobStorage` and provide a container name to use and credentials: + + + +To use account name and access key, add them like this: ```json { ... "Storage": { "Type": "AzureBlobStorage", + "Container": "my-container", "AccountName": "my-account", - "AccessKey": "abcd1234", - "Container": "my-container" + "AccessKey": "abcd1234" }, ... } ``` -Alternatively, you can use a full Azure Storage connection string: + + + + +To use a connection string, add it like this: ```json { @@ -62,24 +73,27 @@ Alternatively, you can use a full Azure Storage connection string: "Storage": { "Type": "AzureBlobStorage", - "ConnectionString": "AccountName=my-account;AccountKey=abcd1234;...", - "Container": "my-container" + "Container": "my-container", + "ConnectionString": "AccountName=my-account;AccountKey=abcd1234;..." }, ... } ``` + + + ### Azure Search -Update the `appsettings.json` file: +Azure Search is currently not available due to significant API changes BaGetter has to make. Once it's available, you can set the search type to `AzureSearch` and provide an account name and API key: ```json { ... "Search": { - "Type": "Azure", + "Type": "AzureSearch", "AccountName": "my-account", "ApiKey": "ABCD1234" }, @@ -116,8 +130,8 @@ You can restore packages by using the following package source: Some helpful guides: -- [Visual Studio](https://docs.microsoft.com/en-us/nuget/consume-packages/install-use-packages-visual-studio#package-sources) -- [NuGet.config](https://docs.microsoft.com/en-us/nuget/reference/nuget-config-file#package-source-sections) +- [Visual Studio](https://docs.microsoft.com/nuget/consume-packages/install-use-packages-visual-studio#package-sources) +- [NuGet.config](https://docs.microsoft.com/nuget/reference/nuget-config-file#package-source-sections) ## Symbol server @@ -125,4 +139,4 @@ You can load symbols by using the following symbol location: `http://localhost:5000/api/download/symbols` -For Visual Studio, please refer to the [Configure Debugging](https://docs.microsoft.com/en-us/visualstudio/debugger/specify-symbol-dot-pdb-and-source-files-in-the-visual-studio-debugger?view=vs-2017#configure-symbol-locations-and-loading-options) guide. +For Visual Studio, please refer to the [Configure Debugging](https://docs.microsoft.com/visualstudio/debugger/specify-symbol-dot-pdb-and-source-files-in-the-visual-studio-debugger?view=vs-2017#configure-symbol-locations-and-loading-options) guide. diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 6a15d1c4a..206dc0f68 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -42,7 +42,7 @@ const config: Config = { // Please change this to your repo. // Remove this to remove the "edit this page" links. editUrl: - 'https://github.com/bagetter/BaGetter/tree/main/', + 'https://github.com/bagetter/BaGetter/tree/main/docs', }, theme: { customCss: './src/css/custom.css', diff --git a/src/BaGetter.Azure/AzureApplicationExtensions.cs b/src/BaGetter.Azure/AzureApplicationExtensions.cs index 0cd1c4633..71a832616 100644 --- a/src/BaGetter.Azure/AzureApplicationExtensions.cs +++ b/src/BaGetter.Azure/AzureApplicationExtensions.cs @@ -1,4 +1,6 @@ using System; +using Azure.Storage; +using Azure.Storage.Blobs; using BaGetter.Azure; using BaGetter.Core; //using Azure.Cosmos.Table; @@ -81,46 +83,39 @@ public static BaGetterApplication AddAzureTableDatabase( public static BaGetterApplication AddAzureBlobStorage(this BaGetterApplication app) { - throw new NotImplementedException(); + app.Services.AddBaGetterOptions(nameof(BaGetterOptions.Storage)); + app.Services.AddTransient(); + app.Services.TryAddTransient(provider => provider.GetRequiredService()); - //app.Services.AddBaGetterOptions(nameof(BaGetterOptions.Storage)); - //app.Services.AddTransient(); - //app.Services.TryAddTransient(provider => provider.GetRequiredService()); + app.Services.AddSingleton(provider => + { + var options = provider.GetRequiredService>().Value; - //app.Services.AddSingleton(provider => - //{ - // var options = provider.GetRequiredService>().Value; - - // if (!string.IsNullOrEmpty(options.ConnectionString)) - // { - // return CloudStorageAccount.Parse(options.ConnectionString); - // } - - // return new CloudStorageAccount( - // new StorageCredentials( - // options.AccountName, - // options.AccessKey), - // useHttps: true); - //}); + // TODO: Add BlobClientOptions with customer-provided key. + if (!string.IsNullOrEmpty(options.ConnectionString)) + { + return new BlobServiceClient(options.ConnectionString); + } - //app.Services.AddTransient(provider => - //{ - // var options = provider.GetRequiredService>().Value; - // var account = provider.GetRequiredService(); + return new BlobServiceClient(new Uri($"https://{options.AccountName}.blob.core.windows.net"), new StorageSharedKeyCredential(options.AccountName, options.AccessKey)); + }); - // var client = account.CreateCloudBlobClient(); + app.Services.AddTransient(provider => + { + var options = provider.GetRequiredService>().Value; + var account = provider.GetRequiredService(); - // return client.GetContainerReference(options.Container); - //}); + return account.GetBlobContainerClient(options.Container); + }); - //app.Services.AddProvider((provider, config) => - //{ - // if (!config.HasStorageType("AzureBlobStorage")) return null; + app.Services.AddProvider((provider, config) => + { + if (!config.HasStorageType("AzureBlobStorage")) return null; - // return provider.GetRequiredService(); - //}); + return provider.GetRequiredService(); + }); - //return app; + return app; } public static BaGetterApplication AddAzureBlobStorage( diff --git a/src/BaGetter.Azure/BaGetter.Azure.csproj b/src/BaGetter.Azure/BaGetter.Azure.csproj index 0d20830f3..a88d1d3e1 100644 --- a/src/BaGetter.Azure/BaGetter.Azure.csproj +++ b/src/BaGetter.Azure/BaGetter.Azure.csproj @@ -10,6 +10,7 @@ + @@ -19,6 +20,7 @@ - + + diff --git a/src/BaGetter.Azure/Configuration/AzureBlobStorageOptions.cs b/src/BaGetter.Azure/Configuration/AzureBlobStorageOptions.cs index 504106193..6e349e05c 100644 --- a/src/BaGetter.Azure/Configuration/AzureBlobStorageOptions.cs +++ b/src/BaGetter.Azure/Configuration/AzureBlobStorageOptions.cs @@ -5,7 +5,7 @@ namespace BaGetter.Azure { /// /// BaGetter's configurations to use Azure Blob Storage to store packages. - /// See: https://loic-sharma.github.io/BaGet/quickstart/azure/#azure-blob-storage + /// See: https://www.bagetter.com/docs/Installation/azure#azure-blob-storage /// public class AzureBlobStorageOptions : IValidatableObject { diff --git a/src/BaGetter.Azure/Extensions/StorageExceptionExtensions.cs b/src/BaGetter.Azure/Extensions/StorageExceptionExtensions.cs index f0baaeb8b..4a06c1f80 100644 --- a/src/BaGetter.Azure/Extensions/StorageExceptionExtensions.cs +++ b/src/BaGetter.Azure/Extensions/StorageExceptionExtensions.cs @@ -1,30 +1,29 @@ +using Azure; using System.Net; namespace BaGetter.Azure { - using StorageException = Microsoft.WindowsAzure.Storage.StorageException; - using TableStorageException = Microsoft.Azure.Cosmos.Table.StorageException; internal static class StorageExceptionExtensions { - public static bool IsAlreadyExistsException(this StorageException e) + public static bool IsAlreadyExistsException(this RequestFailedException e) { - return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict; + return e?.Status == (int?)HttpStatusCode.Conflict; } - public static bool IsNotFoundException(this TableStorageException e) - { - return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.NotFound; - } + //public static bool IsNotFoundException(this TableStorageException e) + //{ + // return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.NotFound; + //} - public static bool IsAlreadyExistsException(this TableStorageException e) - { - return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict; - } + //public static bool IsAlreadyExistsException(this TableStorageException e) + //{ + // return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.Conflict; + //} - public static bool IsPreconditionFailedException(this TableStorageException e) - { - return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.PreconditionFailed; - } + //public static bool IsPreconditionFailedException(this TableStorageException e) + //{ + // return e?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.PreconditionFailed; + //} } } diff --git a/src/BaGetter.Azure/Storage/BlobStorageService.cs b/src/BaGetter.Azure/Storage/BlobStorageService.cs index 1e6a3183f..7a2c1a0e3 100644 --- a/src/BaGetter.Azure/Storage/BlobStorageService.cs +++ b/src/BaGetter.Azure/Storage/BlobStorageService.cs @@ -2,43 +2,40 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Sas; using BaGetter.Core; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; namespace BaGetter.Azure { // See: https://github.com/NuGet/NuGetGallery/blob/master/src/NuGetGallery.Core/Services/CloudBlobCoreFileStorageService.cs public class BlobStorageService : IStorageService { - private readonly CloudBlobContainer _container; + private readonly BlobContainerClient _container; - public BlobStorageService(CloudBlobContainer container) + public BlobStorageService(BlobContainerClient container) { - _container = container ?? throw new ArgumentNullException(nameof(container)); + ArgumentNullException.ThrowIfNull(container); + + _container = container; } public async Task GetAsync(string path, CancellationToken cancellationToken) { + ArgumentException.ThrowIfNullOrEmpty(path); + return await _container - .GetBlockBlobReference(path) - .OpenReadAsync(cancellationToken); + .GetBlobClient(path) + .OpenReadAsync(new(true), cancellationToken); } public Task GetDownloadUriAsync(string path, CancellationToken cancellationToken) { // TODO: Make expiry time configurable. - var blob = _container.GetBlockBlobReference(path); - var accessPolicy = new SharedAccessBlobPolicy - { - SharedAccessExpiryTime = DateTimeOffset.Now.Add(TimeSpan.FromMinutes(10)), - Permissions = SharedAccessBlobPermissions.Read - }; - - var signature = blob.GetSharedAccessSignature(accessPolicy); - var result = new Uri(blob.Uri, signature); - - return Task.FromResult(result); + var blob = _container.GetBlobClient(path); + return Task.FromResult(blob.GenerateSasUri(BlobSasPermissions.Read, DateTimeOffset.Now.AddMinutes(10))); } public async Task PutAsync( @@ -47,39 +44,44 @@ public async Task PutAsync( string contentType, CancellationToken cancellationToken) { - var blob = _container.GetBlockBlobReference(path); - var condition = AccessCondition.GenerateIfNotExistsCondition(); - - blob.Properties.ContentType = contentType; + var blob = _container.GetBlobClient(path); + var condition = new BlobRequestConditions() + { + IfNoneMatch = new ETag("*") + }; try { - await blob.UploadFromStreamAsync( + await blob.UploadAsync( content, - condition, - options: null, - operationContext: null, - cancellationToken: cancellationToken); + new BlobUploadOptions + { + Conditions = condition, + }, + cancellationToken); + + await blob.SetHttpHeadersAsync(new BlobHttpHeaders + { + ContentType = contentType, + }, cancellationToken: cancellationToken); return StoragePutResult.Success; } - catch (StorageException e) when (e.IsAlreadyExistsException()) + catch (RequestFailedException e) when (e.IsAlreadyExistsException()) { - using (var targetStream = await blob.OpenReadAsync(cancellationToken)) - { - content.Position = 0; - return content.Matches(targetStream) - ? StoragePutResult.AlreadyExists - : StoragePutResult.Conflict; - } + await using var targetStream = await blob.OpenReadAsync(new(true), cancellationToken); + content.Position = 0; + return content.Matches(targetStream) + ? StoragePutResult.AlreadyExists + : StoragePutResult.Conflict; } } public async Task DeleteAsync(string path, CancellationToken cancellationToken) { await _container - .GetBlockBlobReference(path) - .DeleteIfExistsAsync(cancellationToken); + .GetBlobClient(path) + .DeleteIfExistsAsync(cancellationToken: cancellationToken); } } } diff --git a/src/BaGetter/Startup.cs b/src/BaGetter/Startup.cs index 520555782..a87533dc2 100644 --- a/src/BaGetter/Startup.cs +++ b/src/BaGetter/Startup.cs @@ -62,7 +62,7 @@ private void ConfigureBaGetterApplication(BaGetterApplication app) app.AddFileStorage(); app.AddAliyunOssStorage(); app.AddAwsS3Storage(); - //app.AddAzureBlobStorage(); + app.AddAzureBlobStorage(); app.AddGoogleCloudStorage(); // Add search providers.