diff --git a/.github/workflows/issues.yaml b/.github/workflows/issues.yaml deleted file mode 100644 index 23d81019c7..0000000000 --- a/.github/workflows/issues.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: Issues Automation - -on: - issues: - types: [opened] - -jobs: - customer: - name: Add a label if Issue created by customer - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GH_RAD_CI_BOT_PAT }} - steps: - - name: Add label if author in preview team - env: - ISSUE: ${{ github.event.issue.html_url }} - AUTHOR: ${{ github.actor }} - run: | - MEMBERS=$(gh api orgs/radius-project/teams/Radius-Preview/members | jq -r '.[] | .login') - ADMINS=$(gh api orgs/radius-project/teams/Radius-Admin/members | jq -r '.[] | .login') - for USER in $MEMBERS; do - if [[ "$USER" == "$AUTHOR" ]]; then - ISADMIN=false - for ADMIN in $ADMINS; do - if [[ "$USER" == "$ADMIN" ]]; then - echo "$USER is an admin, skipping" - ISADMIN=true - fi - done - if [[ "$ISADMIN" == "false" ]]; then - echo "Adding label for issue $ISSUE" - gh issue edit $ISSUE --add-label "customer-issue" - fi - fi - done diff --git a/pkg/recipes/driver/bicep.go b/pkg/recipes/driver/bicep.go index 13459784f0..c1478dad03 100644 --- a/pkg/recipes/driver/bicep.go +++ b/pkg/recipes/driver/bicep.go @@ -25,10 +25,11 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - "github.com/radius-project/radius/pkg/to" + "github.com/go-logr/logr" "golang.org/x/sync/errgroup" + "oras.land/oras-go/v2/registry/remote" - "github.com/go-logr/logr" + coredm "github.com/radius-project/radius/pkg/corerp/datamodel" "github.com/radius-project/radius/pkg/metrics" "github.com/radius-project/radius/pkg/portableresources/datamodel" "github.com/radius-project/radius/pkg/portableresources/processors" @@ -37,12 +38,11 @@ import ( recipes_util "github.com/radius-project/radius/pkg/recipes/util" "github.com/radius-project/radius/pkg/rp/util" rpv1 "github.com/radius-project/radius/pkg/rp/v1" - clients "github.com/radius-project/radius/pkg/sdk/clients" + "github.com/radius-project/radius/pkg/sdk/clients" + "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/resources" resources_radius "github.com/radius-project/radius/pkg/ucp/resources/radius" "github.com/radius-project/radius/pkg/ucp/ucplog" - - coredm "github.com/radius-project/radius/pkg/corerp/datamodel" ) //go:generate mockgen -destination=./mock_driver.go -package=driver -self_package github.com/radius-project/radius/pkg/recipes/driver github.com/radius-project/radius/pkg/recipes/driver Driver @@ -74,6 +74,9 @@ type bicepDriver struct { DeploymentClient *clients.ResourceDeploymentsClient ResourceClient processors.ResourceClient options BicepOptions + + // RegistryClient is the optional client used to interact with the container registry. + RegistryClient remote.Client } // Execute fetches recipe contents from container registry, creates a deployment ID, a recipe context parameter, recipe parameters, @@ -85,7 +88,8 @@ func (d *bicepDriver) Execute(ctx context.Context, opts ExecuteOptions) (*recipe recipeData := make(map[string]any) downloadStartTime := time.Now() - err := util.ReadFromRegistry(ctx, opts.Definition, &recipeData) + + err := util.ReadFromRegistry(ctx, opts.Definition, &recipeData, d.RegistryClient) if err != nil { metrics.DefaultRecipeEngineMetrics.RecordRecipeDownloadDuration(ctx, downloadStartTime, metrics.NewRecipeAttributes(metrics.RecipeEngineOperationDownloadRecipe, opts.Recipe.Name, &opts.Definition, recipes.RecipeDownloadFailed)) @@ -258,7 +262,7 @@ func (d *bicepDriver) GetRecipeMetadata(ctx context.Context, opts BaseOptions) ( // } // } recipeData := make(map[string]any) - err := util.ReadFromRegistry(ctx, opts.Definition, &recipeData) + err := util.ReadFromRegistry(ctx, opts.Definition, &recipeData, d.RegistryClient) if err != nil { return nil, err } diff --git a/pkg/recipes/driver/bicep_test.go b/pkg/recipes/driver/bicep_test.go index cbe9802b81..729c1c86a5 100644 --- a/pkg/recipes/driver/bicep_test.go +++ b/pkg/recipes/driver/bicep_test.go @@ -18,21 +18,24 @@ package driver import ( "fmt" + "strings" "testing" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" - gomock "github.com/golang/mock/gomock" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" corerp_datamodel "github.com/radius-project/radius/pkg/corerp/datamodel" "github.com/radius-project/radius/pkg/portableresources/processors" "github.com/radius-project/radius/pkg/recipes" "github.com/radius-project/radius/pkg/recipes/recipecontext" + "github.com/radius-project/radius/pkg/rp/util/registrytest" rpv1 "github.com/radius-project/radius/pkg/rp/v1" clients "github.com/radius-project/radius/pkg/sdk/clients" "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/resources" resources_kubernetes "github.com/radius-project/radius/pkg/ucp/resources/kubernetes" "github.com/radius-project/radius/test/testcontext" + + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) @@ -376,7 +379,9 @@ func Test_Bicep_PrepareRecipeResponse_EmptyResult(t *testing.T) { } func Test_Bicep_Execute_SimulatedEnvironment(t *testing.T) { - t.Skip("This test makes outbound calls. #6490") + ts := registrytest.NewFakeRegistryServer(t) + t.Cleanup(ts.CloseServer) + opts := ExecuteOptions{ BaseOptions: BaseOptions{ Configuration: recipes.Configuration{ @@ -395,13 +400,13 @@ func Test_Bicep_Execute_SimulatedEnvironment(t *testing.T) { Definition: recipes.EnvironmentDefinition{ Name: "test-recipe", Driver: recipes.TemplateKindBicep, - TemplatePath: "ghcr.io/radius-project/dev/recipes/functionaltest/parameters/mongodatabases/azure:1.0", + TemplatePath: ts.TestImageURL, ResourceType: "Applications.Datastores/mongoDatabases", }, }, } ctx := testcontext.New(t) - d := &bicepDriver{} + d := &bicepDriver{RegistryClient: ts.TestServer.Client()} recipesOutput, err := d.Execute(ctx, opts) require.NoError(t, err) require.Nil(t, recipesOutput) @@ -489,20 +494,21 @@ func Test_Bicep_Delete_Error(t *testing.T) { } func Test_Bicep_GetRecipeMetadata_Success(t *testing.T) { - t.Skip("This test makes outbound calls. #6490") + ts := registrytest.NewFakeRegistryServer(t) + t.Cleanup(ts.CloseServer) + ctx := testcontext.New(t) - driver := bicepDriver{} + driver := &bicepDriver{RegistryClient: ts.TestServer.Client()} recipeDefinition := recipes.EnvironmentDefinition{ Name: "mongo-azure", Driver: recipes.TemplateKindBicep, - TemplatePath: "ghcr.io/radius-project/dev/recipes/functionaltest/parameters/mongodatabases/azure:1.0", + TemplatePath: ts.TestImageURL, ResourceType: "Applications.Datastores/mongoDatabases", } expectedOutput := map[string]any{ "documentdbName": map[string]any{"type": "string"}, "location": map[string]any{"defaultValue": "[resourceGroup().location]", "type": "string"}, - "mongodbName": map[string]any{"type": "string"}, } recipeData, err := driver.GetRecipeMetadata(ctx, BaseOptions{ @@ -515,30 +521,31 @@ func Test_Bicep_GetRecipeMetadata_Success(t *testing.T) { } func Test_Bicep_GetRecipeMetadata_Error(t *testing.T) { - t.Skip("This test makes outbound calls. #6490") + ts := registrytest.NewFakeRegistryServer(t) + t.Cleanup(ts.CloseServer) + ctx := testcontext.New(t) - driver := bicepDriver{} + driver := &bicepDriver{RegistryClient: ts.TestServer.Client()} recipeDefinition := recipes.EnvironmentDefinition{ Name: "mongo-azure", Driver: recipes.TemplateKindBicep, - TemplatePath: "ghcr.io/radius-project/dev/test-non-existent-recipe", + TemplatePath: ts.TestServer.URL + "/nonexisting:latest", ResourceType: "Applications.Datastores/mongoDatabases", } - _, err := driver.GetRecipeMetadata(ctx, BaseOptions{ + _, actualErr := driver.GetRecipeMetadata(ctx, BaseOptions{ Recipe: recipes.ResourceMetadata{}, Definition: recipeDefinition, }) expErr := recipes.RecipeError{ ErrorDetails: v1.ErrorDetails{ Code: recipes.RecipeLanguageFailure, - Message: "failed to fetch repository from the path \"ghcr.io/radius-project/dev/test-non-existent-recipe\": ghcr.io/radius-project/dev/test-non-existent-recipe:latest: not found", + Message: "failed to fetch repository from the path \"https:///nonexisting:latest\": /nonexisting:latest: not found", }, DeploymentStatus: "setupError", } - - require.Error(t, err) - require.Equal(t, err, &expErr) + expErr.ErrorDetails.Message = strings.Replace(expErr.ErrorDetails.Message, "", ts.URL.Host, -1) + require.Equal(t, actualErr, &expErr) } func Test_GetGCOutputResources(t *testing.T) { diff --git a/pkg/rp/util/registry.go b/pkg/rp/util/registry.go index e362f7f6f5..9dc6737f1b 100644 --- a/pkg/rp/util/registry.go +++ b/pkg/rp/util/registry.go @@ -32,7 +32,7 @@ import ( // ReadFromRegistry reads data from an OCI compliant registry and stores it in a map. It returns an error if the path is invalid, // if the client to the registry fails to be created, if the manifest fails to be fetched, if the bytes fail to be fetched, or if // the data fails to be unmarshalled. -func ReadFromRegistry(ctx context.Context, definition recipes.EnvironmentDefinition, data *map[string]any) error { +func ReadFromRegistry(ctx context.Context, definition recipes.EnvironmentDefinition, data *map[string]any, client remote.Client) error { registryRepo, tag, err := parsePath(definition.TemplatePath) if err != nil { return v1.NewClientErrInvalidRequest(fmt.Sprintf("invalid path %s", err.Error())) @@ -42,6 +42,7 @@ func ReadFromRegistry(ctx context.Context, definition recipes.EnvironmentDefinit if err != nil { return fmt.Errorf("failed to create client to registry %s", err.Error()) } + repo.Client = client if definition.PlainHTTP { repo.PlainHTTP = true @@ -88,6 +89,7 @@ func getDigestFromManifest(ctx context.Context, repo *remote.Repository, tag str if err != nil { return "", err } + // get the layers digest to fetch the blob layer, ok := manifest["layers"].([]any)[0].(map[string]any) if !ok { diff --git a/pkg/rp/util/registrytest/fakeregistryserver.go b/pkg/rp/util/registrytest/fakeregistryserver.go new file mode 100644 index 0000000000..988a96c1ce --- /dev/null +++ b/pkg/rp/util/registrytest/fakeregistryserver.go @@ -0,0 +1,120 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registrytest + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + + "github.com/go-chi/chi/v5" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type fakeServerInfo struct { + TestServer *httptest.Server + URL *url.URL + CloseServer func() + TestImageURL string + ImageName string +} + +// NewFakeRegistryServer creates a fake registry server that serves a single blob and index. +func NewFakeRegistryServer(t *testing.T) fakeServerInfo { + blob := []byte(`{ + "parameters": { + "documentdbName": { + "type": "string" + }, + "location": { + "defaultValue": "[resourceGroup().location]", + "type": "string" + } + } +}`) + + blobDesc := ocispec.Descriptor{ + MediaType: "recipe", + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + } + + index := []byte(`{ + "layers": [ + { + "digest": "` + blobDesc.Digest.String() + `" + } + ] +}`) + + indexDesc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageIndex, + Digest: digest.FromBytes(index), + Size: int64(len(index)), + } + + r := chi.NewRouter() + r.Route("/v2/test", func(r chi.Router) { + r.Head("/manifests/{ref}", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", indexDesc.MediaType) + w.Header().Set("Docker-Content-Digest", indexDesc.Digest.String()) + w.Header().Set("Content-Length", strconv.Itoa(int(indexDesc.Size))) + w.WriteHeader(http.StatusOK) + }) + + r.Get("/manifests/"+indexDesc.Digest.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", indexDesc.MediaType) + w.Header().Set("Docker-Content-Digest", indexDesc.Digest.String()) + if _, err := w.Write(index); err != nil { + t.Errorf("failed to write %q: %v", r.URL, err) + } + }) + + r.Head("/blobs/{digest}", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", blobDesc.MediaType) + w.Header().Set("Docker-Content-Digest", blobDesc.Digest.String()) + w.Header().Set("Content-Length", strconv.Itoa(int(blobDesc.Size))) + w.WriteHeader(http.StatusOK) + }) + + r.Get("/blobs/"+blobDesc.Digest.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Docker-Content-Digest", blobDesc.Digest.String()) + if _, err := w.Write(blob); err != nil { + t.Errorf("failed to write %q: %v", r.URL, err) + } + }) + }) + + ts := httptest.NewTLSServer(r) + + url, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("failed to parse url: %v", err) + } + + return fakeServerInfo{ + TestServer: ts, + URL: url, + CloseServer: ts.Close, + TestImageURL: ts.URL + "/test:latest", + ImageName: "test:latest", + } +}