forked from k8sgpt-ai/k8sgpt
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add Azure remote cache (k8sgpt-ai#690)
* feat: add Azure remote cache Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * feat: add serve mode support and update buf schema Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix: map structure name Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * chore: add a new cache type to make code readable Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * docs: update docs to reflect new remote cache type Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module github.com/prometheus/client_golang to v1.17.0 (k8sgpt-ai#687) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go to v1.3.0-20231002095256-194bc640518b.1 (k8sgpt-ai#692) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(deps): update module helm.sh/helm/v3 to v3.13.0 (k8sgpt-ai#688) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix: security warning around printing provider details in https://github.com/k8sgpt-ai/k8sgpt/security/code-scanning/1 (k8sgpt-ai#695) Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go to v1.31.0-20231002095256-194bc640518b.1 (k8sgpt-ai#693) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(deps): update module github.com/sashabaranov/go-openai to v1.15.4 (k8sgpt-ai#689) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.20 (k8sgpt-ai#685) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * chore(deps): update amannn/action-semantic-pull-request action to v5.3.0 (k8sgpt-ai#683) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.21 (k8sgpt-ai#696) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.22 (k8sgpt-ai#697) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.23 (k8sgpt-ai#699) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.24 (k8sgpt-ai#701) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> --------- Signed-off-by: Aris Boutselis <arisboutselis08@gmail.com> Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis <aris.boutselis@senseon.io> Co-authored-by: Aris Boutselis <arisboutselis08@gmail.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Alex Jones <alexsimonjones@gmail.com>
- Loading branch information
4 people
authored
Oct 22, 2023
1 parent
c977528
commit 23ac52d
Showing
9 changed files
with
268 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| package cache | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "log" | ||
| "strings" | ||
|
|
||
| "github.com/Azure/azure-sdk-for-go/sdk/azidentity" | ||
| "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" | ||
| "github.com/spf13/viper" | ||
| ) | ||
|
|
||
| // Generate ICache implementation | ||
| type AzureCache struct { | ||
| ctx context.Context | ||
| noCache bool | ||
| containerName string | ||
| session *azblob.Client | ||
| } | ||
|
|
||
| func (s *AzureCache) Store(key string, data string) error { | ||
| // Store the object as a new file in the Azure blob storage with data as the content | ||
| cacheData := []byte(data) | ||
| _, err := s.session.UploadBuffer(s.ctx, s.containerName, key, cacheData, &azblob.UploadBufferOptions{}) | ||
| return err | ||
| } | ||
|
|
||
| func (s *AzureCache) Load(key string) (string, error) { | ||
| // Load blob file contents | ||
| load, err := s.session.DownloadStream(s.ctx, s.containerName, key, nil) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| data := bytes.Buffer{} | ||
| retryReader := load.NewRetryReader(s.ctx, &azblob.RetryReaderOptions{}) | ||
| _, err = data.ReadFrom(retryReader) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| if err := retryReader.Close(); err != nil { | ||
| return "", err | ||
| } | ||
| return data.String(), nil | ||
| } | ||
|
|
||
| func (s *AzureCache) List() ([]string, error) { | ||
| // List the files in the blob containerName | ||
| files := []string{} | ||
|
|
||
| pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{ | ||
| Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false}, | ||
| }) | ||
|
|
||
| for pager.More() { | ||
| resp, err := pager.NextPage(s.ctx) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| for _, blob := range resp.Segment.BlobItems { | ||
| files = append(files, *blob.Name) | ||
| } | ||
| } | ||
|
|
||
| return files, nil | ||
| } | ||
|
|
||
| func (s *AzureCache) Exists(key string) bool { | ||
| // Check if the object exists in the blob storage | ||
| pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{ | ||
| Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false}, | ||
| }) | ||
|
|
||
| for pager.More() { | ||
| resp, err := pager.NextPage(s.ctx) | ||
| if err != nil { | ||
| return false | ||
| } | ||
|
|
||
| for _, blob := range resp.Segment.BlobItems { | ||
| if *blob.Name == key { | ||
| return true | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return false | ||
| } | ||
|
|
||
| func (s *AzureCache) IsCacheDisabled() bool { | ||
| return s.noCache | ||
| } | ||
|
|
||
| func NewAzureCache(nocache bool) ICache { | ||
| ctx := context.Background() | ||
| var cache CacheProvider | ||
| err := viper.UnmarshalKey("cache", &cache) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| if cache.ContainerName == "" { | ||
| log.Fatal("Azure Container name not configured") | ||
| } | ||
| if cache.StorageAccount == "" { | ||
| log.Fatal("Azure Storage account not configured") | ||
| } | ||
|
|
||
| // We assume that Storage account is already in place | ||
| blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cache.StorageAccount) | ||
| credential, err := azidentity.NewDefaultAzureCredential(nil) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| client, err := azblob.NewClient(blobUrl, credential, nil) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| // Try to create the blob container | ||
| _, err = client.CreateContainer(ctx, cache.ContainerName, nil) | ||
| if err != nil { | ||
| // TODO: Maybe there is a better way to check this? | ||
| // docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob | ||
| if strings.Contains(err.Error(), "ContainerAlreadyExists") { | ||
| // do nothing | ||
| } else { | ||
| log.Fatal(err) | ||
| } | ||
| } | ||
|
|
||
| return &AzureCache{ | ||
| ctx: ctx, | ||
| noCache: nocache, | ||
| containerName: cache.ContainerName, | ||
| session: client, | ||
| } | ||
| } |
Oops, something went wrong.