fix(controlplane): fallback CAS downloads from invalid backends

Use the organization's current default or fallback backend when a historical
CAS mapping points to an invalid backend.

Fixes: #3190

Assisted-by: pi
Signed-off-by: Vibhav Bobade <vibhav.bobde@gmail.com>

Author: waveywaves

app/controlplane/cmd/wire_gen.go

@@ -130,7 +130,10 @@ func (s *CASCredentialsService) Get(ctx context.Context, req *pb.CASCredentialsS
 		}
 
 		if mapping != nil {
-			backend = mapping.CASBackend
+			backend, err = s.casBackendUC.FindDownloadBackend(ctx, mapping.CASBackend)
+			if err != nil {
+				return nil, handleUseCaseErr(err, s.log)
+			}
 		} else {
 			// fallback to default backend if the user or the token is allowed to
 			if ok, err := s.authzUC.Enforce(ctx, currentAuthzSubject, authz.PolicyDefaultBackendArtifactRead); err != nil {

app/controlplane/internal/service/cascredential.go

@@ -49,10 +49,11 @@ type CASRedirectService struct {
 
 	casMappingUC    *biz.CASMappingUseCase
 	casCredsUseCase *biz.CASCredentialsUseCase
+	casBackendUC    *biz.CASBackendUseCase
 	casServerConf   *conf.Bootstrap_CASServer
 }
 
-func NewCASRedirectService(casmUC *biz.CASMappingUseCase, casCredsUC *biz.CASCredentialsUseCase, conf *conf.Bootstrap_CASServer, opts ...NewOpt) (*CASRedirectService, error) {
+func NewCASRedirectService(casmUC *biz.CASMappingUseCase, casCredsUC *biz.CASCredentialsUseCase, casBackendUC *biz.CASBackendUseCase, conf *conf.Bootstrap_CASServer, opts ...NewOpt) (*CASRedirectService, error) {
 	if conf == nil || conf.GetDownloadUrl() == "" {
 		return nil, errors.New("CASServer.downloadURL configuration is missing")
 	}
@@ -61,6 +62,7 @@ func NewCASRedirectService(casmUC *biz.CASMappingUseCase, casCredsUC *biz.CASCre
 		service:         newService(opts...),
 		casMappingUC:    casmUC,
 		casCredsUseCase: casCredsUC,
+		casBackendUC:    casBackendUC,
 		casServerConf:   conf,
 	}, nil
 }
@@ -103,18 +105,16 @@ func (s *CASRedirectService) GetDownloadURL(ctx context.Context, req *pb.GetDown
 		return nil, handleUseCaseErr(err, s.log)
 	}
 
-	backend := mapping.CASBackend
+	backend, err := s.casBackendUC.FindDownloadBackend(ctx, mapping.CASBackend)
+	if err != nil {
+		return nil, handleUseCaseErr(err, s.log)
+	}
 
 	// inline backends don't have a download URL
 	if backend.Inline {
 		return nil, kerrors.NotFound("not found", "CAS backend is inline")
 	}
 
-	// check if the backend is on a valid state, if not return an error
-	if backend.ValidationStatus != biz.CASBackendValidationOK {
-		return nil, pb.ErrorCasBackendErrorReasonInvalid("CAS Storage is in an invalid state and can't download artifacts, please fix it before attempting it again")
-	}
-
 	// Create an URL to download the artifact from the CAS backend
 	downloadBase, err := url.Parse(s.casServerConf.GetDownloadUrl())
 	if err != nil {

app/controlplane/internal/service/casredirect.go

@@ -350,6 +350,30 @@ func (uc *CASBackendUseCase) FindDefaultOrFallbackBackend(ctx context.Context, o
 	return fallbackBackend, nil
 }
 
+// FindDownloadBackend returns the backend that should serve a mapped artifact download.
+//
+// CAS mappings are historical: digest -> backend ID. If credentials are rotated by
+// creating a new backend for the same storage location and making it the org default,
+// old mappings still point at the backend with revoked credentials.
+//
+// For that case, try the org's current default backend first, then the configured
+// fallback. This is safe because CAS downloads are verified by digest: if the chosen
+// backend does not contain the exact content, the download fails.
+//
+// The stored mapping is not changed; this only chooses credentials for this read.
+func (uc *CASBackendUseCase) FindDownloadBackend(ctx context.Context, mapped *CASBackend) (*CASBackend, error) {
+	if mapped == nil {
+		return nil, NewErrNotFound("CAS Backend")
+	}
+
+	if mapped.ValidationStatus == CASBackendValidationOK {
+		return mapped, nil
+	}
+
+	uc.logger.Infow("msg", "mapped CAS backend validation failed, attempting default/fallback", "backend", mapped.Name, "status", mapped.ValidationStatus, "orgID", mapped.OrganizationID)
+	return uc.FindDefaultOrFallbackBackend(ctx, mapped.OrganizationID.String())
+}
+
 func (uc *CASBackendUseCase) CreateInlineBackend(ctx context.Context, orgID string) (*CASBackend, error) {
 	ctx, span := otelx.Start(ctx, casBackendTracer, "CASBackendUseCase.CreateInlineBackend")
 	defer span.End()

app/controlplane/pkg/biz/casbackend.go

@@ -72,6 +72,62 @@ func (s *casBackendTestSuite) TestFindDefaultBackendFound() {
 	assert.Equal(backend, wantBackend)
 }
 
+func (s *casBackendTestSuite) TestFindDownloadBackend() {
+	ctx := context.Background()
+	mapped := &biz.CASBackend{ID: uuid.New(), OrganizationID: s.validUUID, ValidationStatus: biz.CASBackendValidationOK}
+
+	got, err := s.useCase.FindDownloadBackend(ctx, mapped)
+	s.Require().NoError(err)
+	s.Same(mapped, got)
+
+	s.resetMock()
+	mapped.ValidationStatus = biz.CASBackendValidationFailed
+	defaultBackend := &biz.CASBackend{ID: uuid.New(), ValidationStatus: biz.CASBackendValidationOK}
+	s.repo.On("FindDefaultBackend", mock.Anything, s.validUUID).Return(defaultBackend, nil).Once()
+
+	got, err = s.useCase.FindDownloadBackend(ctx, mapped)
+	s.Require().NoError(err)
+	s.Same(defaultBackend, got)
+
+	s.resetMock()
+	defaultBackend.ValidationStatus = biz.CASBackendValidationFailed
+	fallbackBackend := &biz.CASBackend{ID: uuid.New(), ValidationStatus: biz.CASBackendValidationOK}
+	s.repo.On("FindDefaultBackend", mock.Anything, s.validUUID).Return(defaultBackend, nil).Once()
+	s.repo.On("FindFallbackBackend", mock.Anything, s.validUUID).Return(fallbackBackend, nil).Once()
+
+	got, err = s.useCase.FindDownloadBackend(ctx, mapped)
+	s.Require().NoError(err)
+	s.Same(fallbackBackend, got)
+
+	got, err = s.useCase.FindDownloadBackend(ctx, nil)
+	s.Nil(got)
+	s.True(biz.IsNotFound(err))
+
+	s.resetMock()
+	s.repo.On("FindDefaultBackend", mock.Anything, s.validUUID).Return(nil, nil).Once()
+
+	got, err = s.useCase.FindDownloadBackend(ctx, mapped)
+	s.Nil(got)
+	s.True(biz.IsNotFound(err))
+
+	s.resetMock()
+	s.repo.On("FindDefaultBackend", mock.Anything, s.validUUID).Return(defaultBackend, nil).Once()
+	s.repo.On("FindFallbackBackend", mock.Anything, s.validUUID).Return(nil, nil).Once()
+
+	got, err = s.useCase.FindDownloadBackend(ctx, mapped)
+	s.Nil(got)
+	s.True(biz.IsNotFound(err))
+
+	s.resetMock()
+	fallbackBackend.ValidationStatus = biz.CASBackendValidationFailed
+	s.repo.On("FindDefaultBackend", mock.Anything, s.validUUID).Return(defaultBackend, nil).Once()
+	s.repo.On("FindFallbackBackend", mock.Anything, s.validUUID).Return(fallbackBackend, nil).Once()
+
+	got, err = s.useCase.FindDownloadBackend(ctx, mapped)
+	s.Nil(got)
+	s.True(biz.IsErrValidation(err))
+}
+
 func (s *casBackendTestSuite) TestSaveInvalidUUID() {
 	repo, err := s.useCase.CreateOrUpdate(context.Background(), s.invalidUUID, "", "", "", backendType, true)
 	assert.True(s.T(), biz.IsErrInvalidUUID(err))