Merge branch 'main' into fix_nightly_version_step

Author: brandtkeller

site/src/content/docs/commands/zarf_tools_registry_prune.md

@@ -17,16 +17,16 @@ zarf tools registry prune [flags]
 ### Options
 
 ```
-  -c, --confirm   Confirm the image prune action to prevent accidental deletions
-  -h, --help      help for prune
+  -c, --confirm    Confirm the image prune action to prevent accidental deletions
+  -h, --help       help for prune
+      --insecure   Allow image references to be fetched without TLS
 ```
 
 ### Options inherited from parent commands
 
 ```
       --allow-nondistributable-artifacts   Allow pushing non-distributable (foreign) layers
       --features stringToString            [ALPHA] Provide a comma-separated list of feature names to bools to enable or disable. Ex. --features "foo=true,bar=false,baz=true" (default [])
-      --insecure                           Allow image references to be fetched without TLS
       --insecure-skip-tls-verify           Skip checking server's certificate for validity. This flag should only be used if you have a specific reason and accept the reduced security posture.
       --plain-http                         Force the connections over HTTP instead of HTTPS. This flag should only be used if you have a specific reason and accept the reduced security posture.
       --platform string                    Specifies the platform in the form os/arch[/variant][:osversion] (e.g. linux/amd64). (default "all")

src/cmd/crane.go

@@ -82,8 +82,8 @@ func newRegistryCommand() *cobra.Command {
 
 	cmd.AddCommand(newRegistryPruneCommand())
 	cmd.AddCommand(newRegistryLoginCommand())
-	cmd.AddCommand(newRegistryCopyCommand())
-	cmd.AddCommand(newRegistryCatalogCommand())
+	cmd.AddCommand(newRegistryCopyCommand(&craneOptions))
+	cmd.AddCommand(newRegistryCatalogCommand(&craneOptions))
 
 	// TODO(soltysh): consider splitting craneOptions to be per command
 	cmd.AddCommand(zarfCraneInternalWrapper(craneCmd.NewCmdList, &craneOptions, lang.CmdToolsRegistryListExample, 0))
@@ -115,25 +115,35 @@ func newRegistryLoginCommand() *cobra.Command {
 	return cmd
 }
 
-func newRegistryCopyCommand() *cobra.Command {
-	// No package information is available so do not pass in a list of architectures
-	craneOptions := []crane.Option{}
-	cmd := craneCmd.NewCmdCopy(&craneOptions)
+func newRegistryCopyCommand(craneOpts *[]crane.Option) *cobra.Command {
+	cmd := craneCmd.NewCmdCopy(craneOpts)
+	// Store crane's original PreRunE if it exists
+	originalPreRunE := cmd.PreRunE
+	cmd.PreRunE = func(cmd *cobra.Command, args []string) error {
+		// No package information is available so do not pass in a list of architectures
+		*craneOpts = append(*craneOpts, crane.WithPlatform(nil))
+		// return the original pre run
+		if originalPreRunE != nil {
+			return originalPreRunE(cmd, args)
+		}
+		return nil
+	}
+
 	return cmd
 }
 
 type registryCatalogOptions struct {
-	craneOptions  []crane.Option
+	craneOptions  *[]crane.Option
 	originalRunFn func(cmd *cobra.Command, args []string) error
 }
 
-func newRegistryCatalogCommand() *cobra.Command {
+func newRegistryCatalogCommand(craneOpts *[]crane.Option) *cobra.Command {
 	o := registryCatalogOptions{
 		// No package information is available so do not pass in a list of architectures
-		craneOptions: []crane.Option{},
+		craneOptions: craneOpts,
 	}
 
-	cmd := craneCmd.NewCmdCatalog(&o.craneOptions)
+	cmd := craneCmd.NewCmdCatalog(o.craneOptions)
 	cmd.Example = lang.CmdToolsRegistryCatalogExample
 	cmd.Args = nil
 
@@ -169,7 +179,7 @@ func (o *registryCatalogOptions) run(cmd *cobra.Command, args []string) error {
 
 	// Add the correct authentication to the crane command options
 	authOption := images.WithPullAuth(zarfState.RegistryInfo)
-	o.craneOptions = append(o.craneOptions, authOption)
+	*o.craneOptions = append(*o.craneOptions, authOption)
 
 	if tunnel != nil {
 		defer tunnel.Close()
@@ -179,7 +189,9 @@ func (o *registryCatalogOptions) run(cmd *cobra.Command, args []string) error {
 	return o.originalRunFn(cmd, []string{registryEndpoint})
 }
 
-type registryPruneOptions struct{}
+type registryPruneOptions struct {
+	insecure bool
+}
 
 func newRegistryPruneCommand() *cobra.Command {
 	o := registryPruneOptions{}
@@ -193,6 +205,7 @@ func newRegistryPruneCommand() *cobra.Command {
 
 	// Always require confirm flag (no viper)
 	cmd.Flags().BoolVarP(&config.CommonOptions.Confirm, "confirm", "c", false, lang.CmdToolsRegistryPruneFlagConfirm)
+	cmd.PersistentFlags().BoolVar(&o.insecure, "insecure", false, lang.CmdToolsRegistryFlagInsecure)
 
 	return cmd
 }
@@ -203,6 +216,10 @@ func (o *registryPruneOptions) run(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
+	options := []crane.Option{}
+	if o.insecure {
+		options = append(options, crane.Insecure)
+	}
 
 	ctx := cmd.Context()
 	l := logger.From(ctx)
@@ -226,15 +243,17 @@ func (o *registryPruneOptions) run(cmd *cobra.Command, _ []string) error {
 	if tunnel != nil {
 		l.Info("opening a tunnel to the Zarf registry", "local-endpoint", registryEndpoint, "cluster-address", zarfState.RegistryInfo.Address)
 		defer tunnel.Close()
-		return tunnel.Wrap(func() error { return doPruneImagesForPackages(ctx, zarfState, zarfPackages, registryEndpoint) })
+		return tunnel.Wrap(func() error {
+			return doPruneImagesForPackages(ctx, options, zarfState, zarfPackages, registryEndpoint)
+		})
 	}
 
-	return doPruneImagesForPackages(ctx, zarfState, zarfPackages, registryEndpoint)
+	return doPruneImagesForPackages(ctx, options, zarfState, zarfPackages, registryEndpoint)
 }
 
-func doPruneImagesForPackages(ctx context.Context, s *state.State, zarfPackages []state.DeployedPackage, registryEndpoint string) error {
+func doPruneImagesForPackages(ctx context.Context, options []crane.Option, s *state.State, zarfPackages []state.DeployedPackage, registryEndpoint string) error {
 	l := logger.From(ctx)
-	authOption := images.WithPushAuth(s.RegistryInfo)
+	options = append(options, images.WithPushAuth(s.RegistryInfo))
 
 	l.Info("finding images to prune")
 
@@ -255,7 +274,7 @@ func doPruneImagesForPackages(ctx context.Context, s *state.State, zarfPackages
 						return err
 					}
 
-					digest, err := crane.Digest(transformedImageNoCheck, authOption)
+					digest, err := crane.Digest(transformedImageNoCheck, options...)
 					if err != nil {
 						return err
 					}
@@ -266,20 +285,20 @@ func doPruneImagesForPackages(ctx context.Context, s *state.State, zarfPackages
 	}
 
 	// Find which images and tags are in the registry currently
-	imageCatalog, err := crane.Catalog(registryEndpoint, authOption)
+	imageCatalog, err := crane.Catalog(registryEndpoint, options...)
 	if err != nil {
 		return err
 	}
 	referenceToDigest := map[string]string{}
 	for _, image := range imageCatalog {
 		imageRef := fmt.Sprintf("%s/%s", registryEndpoint, image)
-		tags, err := crane.ListTags(imageRef, authOption)
+		tags, err := crane.ListTags(imageRef, options...)
 		if err != nil {
 			return err
 		}
 		for _, tag := range tags {
 			taggedImageRef := fmt.Sprintf("%s:%s", imageRef, tag)
-			digest, err := crane.Digest(taggedImageRef, authOption)
+			digest, err := crane.Digest(taggedImageRef, options...)
 			if err != nil {
 				return err
 			}
@@ -324,7 +343,7 @@ func doPruneImagesForPackages(ctx context.Context, s *state.State, zarfPackages
 
 		// Delete the digest references that are to be pruned
 		for digestRef := range imageDigestsToPrune {
-			err = crane.Delete(digestRef, authOption)
+			err = crane.Delete(digestRef, options...)
 			if err != nil {
 				return err
 			}

src/internal/packager/helm/chart.go

@@ -77,12 +77,12 @@ func InstallOrUpgradeChart(ctx context.Context, zarfChart v1alpha1.ZarfChart, ch
 	// Setup K8s connection.
 	actionConfig, err := createActionConfig(ctx, zarfChart.Namespace)
 	if err != nil {
-		return nil, "", fmt.Errorf("unable to initialize the K8s client: %w", err)
+		return nil, zarfChart.ReleaseName, fmt.Errorf("unable to initialize the K8s client: %w", err)
 	}
 
 	postRender, err := newRenderer(ctx, zarfChart, opts.AdoptExistingResources, opts.Cluster, opts.AirgapMode, opts.State, actionConfig, opts.VariableConfig)
 	if err != nil {
-		return nil, "", fmt.Errorf("unable to create helm renderer: %w", err)
+		return nil, zarfChart.ReleaseName, fmt.Errorf("unable to create helm renderer: %w", err)
 	}
 
 	histClient := action.NewHistory(actionConfig)
@@ -126,7 +126,7 @@ func InstallOrUpgradeChart(ctx context.Context, zarfChart v1alpha1.ZarfChart, ch
 
 		releases, err := histClient.Run(zarfChart.ReleaseName)
 		if err != nil {
-			return nil, "", errors.Join(err, installErr)
+			return nil, zarfChart.ReleaseName, errors.Join(err, installErr)
 		}
 		previouslyDeployedVersion := 0
 
@@ -139,21 +139,21 @@ func InstallOrUpgradeChart(ctx context.Context, zarfChart v1alpha1.ZarfChart, ch
 
 		// No prior releases means this was an initial install.
 		if previouslyDeployedVersion == 0 {
-			return nil, "", installErr
+			return nil, zarfChart.ReleaseName, installErr
 		}
 
 		// Attempt to rollback on a failed upgrade.
 		l.Info("performing Helm rollback", "chart", zarfChart.Name)
 		err = rollbackChart(zarfChart.ReleaseName, previouslyDeployedVersion, actionConfig, opts.Timeout)
 		if err != nil {
-			return nil, "", fmt.Errorf("%w: unable to rollback: %w", installErr, err)
+			return nil, zarfChart.ReleaseName, fmt.Errorf("%w: unable to rollback: %w", installErr, err)
 		}
-		return nil, "", installErr
+		return nil, zarfChart.ReleaseName, installErr
 	}
 
 	resourceList, err := actionConfig.KubeClient.Build(bytes.NewBufferString(release.Manifest), true)
 	if err != nil {
-		return nil, "", fmt.Errorf("unable to build the resource list: %w", err)
+		return nil, zarfChart.ReleaseName, fmt.Errorf("unable to build the resource list: %w", err)
 	}
 
 	runtimeObjs := []runtime.Object{}
@@ -164,7 +164,7 @@ func InstallOrUpgradeChart(ctx context.Context, zarfChart v1alpha1.ZarfChart, ch
 		// Ensure we don't go past the timeout by using a context initialized with the helm timeout
 		l.Info("running health checks", "chart", zarfChart.Name)
 		if err := healthchecks.WaitForReadyRuntime(helmCtx, opts.Cluster.Watcher, runtimeObjs); err != nil {
-			return nil, "", err
+			return nil, zarfChart.ReleaseName, err
 		}
 	}
 	l.Debug("done processing Helm chart", "name", zarfChart.Name, "duration", time.Since(start))

src/pkg/packager/deploy.go

@@ -221,18 +221,30 @@ func (d *deployer) deployComponents(ctx context.Context, pkgLayout *layout.Packa
 		}
 
 		if deployErr != nil {
-			onFailure()
-			deployedComponents[idx].Status = state.ComponentStatusFailed
-			if d.isConnectedToCluster() {
-				if _, err := d.c.RecordPackageDeployment(ctx, pkgLayout.Pkg, deployedComponents, packageGeneration, state.WithPackageNamespaceOverride(opts.NamespaceOverride)); err != nil {
-					l.Debug("unable to record package deployment", "component", component.Name, "error", err.Error())
+			cleanup := func(ctx context.Context) {
+				onFailure()
+				l.Debug("component deployment failed", "component", component.Name, "error", deployErr.Error())
+				deployedComponents[idx].Status = state.ComponentStatusFailed
+				deployedComponents[idx].InstalledCharts = state.MergeInstalledChartsForComponent(deployedComponents[idx].InstalledCharts, charts, true)
+				if d.isConnectedToCluster() {
+					if _, err := d.c.RecordPackageDeployment(ctx, pkgLayout.Pkg, deployedComponents, packageGeneration, state.WithPackageNamespaceOverride(opts.NamespaceOverride)); err != nil {
+						l.Debug("unable to record package deployment", "component", component.Name, "error", err.Error())
+					}
 				}
 			}
-			return nil, fmt.Errorf("unable to deploy component %q: %w", component.Name, deployErr)
+			select {
+			case <-ctx.Done():
+				// Use background context here in order to ensure the cleanup logic can run when the context is cancelled
+				cleanup(context.Background())
+				return nil, fmt.Errorf("context cancelled while deploying component %q: %w", component.Name, deployErr)
+			default:
+				cleanup(ctx)
+				return nil, fmt.Errorf("unable to deploy component %q: %w", component.Name, deployErr)
+			}
 		}
 
 		// Update the package secret to indicate that we successfully deployed this component
-		deployedComponents[idx].InstalledCharts = charts
+		deployedComponents[idx].InstalledCharts = state.MergeInstalledChartsForComponent(deployedComponents[idx].InstalledCharts, charts, false)
 		deployedComponents[idx].Status = state.ComponentStatusSucceeded
 		if d.isConnectedToCluster() {
 			if _, err := d.c.RecordPackageDeployment(ctx, pkgLayout.Pkg, deployedComponents, packageGeneration, state.WithPackageNamespaceOverride(opts.NamespaceOverride)); err != nil {
@@ -421,35 +433,35 @@ func (d *deployer) deployComponent(ctx context.Context, pkgLayout *layout.Packag
 	charts := []state.InstalledChart{}
 	if hasCharts {
 		helmCharts, err := d.installCharts(ctx, pkgLayout, component, opts)
+		charts = append(charts, helmCharts...)
 		if err != nil {
-			return nil, err
+			return charts, err
 		}
-		charts = append(charts, helmCharts...)
 	}
 
 	if hasManifests {
 		chartsFromManifests, err := d.installManifests(ctx, pkgLayout, component, opts)
+		charts = append(charts, chartsFromManifests...)
 		if err != nil {
-			return nil, err
+			return charts, err
 		}
-		charts = append(charts, chartsFromManifests...)
 	}
 
 	if err := actions.Run(ctx, cwd, onDeploy.Defaults, onDeploy.After, d.vc); err != nil {
-		return nil, fmt.Errorf("unable to run component after action: %w", err)
+		return charts, fmt.Errorf("unable to run component after action: %w", err)
 	}
 
 	if len(component.HealthChecks) > 0 {
 		healthCheckContext, cancel := context.WithTimeout(ctx, opts.Timeout)
 		defer cancel()
 		l.Info("running health checks")
 		if err := healthchecks.Run(healthCheckContext, d.c.Watcher, component.HealthChecks); err != nil {
-			return nil, fmt.Errorf("health checks failed: %w", err)
+			return charts, fmt.Errorf("health checks failed: %w", err)
 		}
 	}
 
 	if err := g.Wait(); err != nil {
-		return nil, err
+		return charts, err
 	}
 	l.Debug("done deploying component", "name", component.Name, "duration", time.Since(start))
 	return charts, nil
@@ -485,15 +497,15 @@ func (d *deployer) installCharts(ctx context.Context, pkgLayout *layout.PackageL
 		for idx := range chart.ValuesFiles {
 			valueFilePath := helm.StandardValuesName(valuesDir, chart, idx)
 			if err := d.vc.ReplaceTextTemplate(valueFilePath); err != nil {
-				return nil, err
+				return installedCharts, err
 			}
 		}
 
 		// Create a Helm values overrides map from set Zarf `variables` and DeployOpts library inputs
 		// Values overrides are to be applied in order of Helm Chart Defaults -> Zarf `valuesFiles` -> Zarf `variables` -> DeployOpts overrides
 		valuesOverrides, err := generateValuesOverrides(chart, component.Name, d.vc, opts.ValuesOverridesMap)
 		if err != nil {
-			return nil, err
+			return installedCharts, err
 		}
 
 		helmOpts := helm.InstallUpgradeOptions{
@@ -507,14 +519,15 @@ func (d *deployer) installCharts(ctx context.Context, pkgLayout *layout.PackageL
 		}
 		helmChart, values, err := helm.LoadChartData(chart, chartDir, valuesDir, valuesOverrides)
 		if err != nil {
-			return nil, fmt.Errorf("failed to load chart data: %w", err)
+			return installedCharts, fmt.Errorf("failed to load chart data: %w", err)
 		}
 
 		connectStrings, installedChartName, err := helm.InstallOrUpgradeChart(ctx, chart, helmChart, values, helmOpts)
 		if err != nil {
-			return nil, err
+			installedCharts = append(installedCharts, state.InstalledChart{Namespace: chart.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings, Status: state.ChartStatusFailed})
+			return installedCharts, err
 		}
-		installedCharts = append(installedCharts, state.InstalledChart{Namespace: chart.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings})
+		installedCharts = append(installedCharts, state.InstalledChart{Namespace: chart.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings, Status: state.ChartStatusSucceeded})
 	}
 
 	return installedCharts, nil
@@ -540,7 +553,7 @@ func (d *deployer) installManifests(ctx context.Context, pkgLayout *layout.Packa
 				// The path is likely invalid because of how we compose OCI components, add an index suffix to the filename
 				manifest.Files[idx] = fmt.Sprintf("%s-%d.yaml", manifest.Name, idx)
 				if helpers.InvalidPath(filepath.Join(manifestDir, manifest.Files[idx])) {
-					return nil, fmt.Errorf("unable to find manifest file %s", manifest.Files[idx])
+					return installedCharts, fmt.Errorf("unable to find manifest file %s", manifest.Files[idx])
 				}
 			}
 		}
@@ -558,7 +571,7 @@ func (d *deployer) installManifests(ctx context.Context, pkgLayout *layout.Packa
 		// Create a helmChart and helm cfg from a given Zarf Manifest.
 		chart, helmChart, err := helm.ChartFromZarfManifest(manifest, manifestDir, pkgLayout.Pkg.Metadata.Name, component.Name)
 		if err != nil {
-			return nil, err
+			return installedCharts, err
 		}
 		helmOpts := helm.InstallUpgradeOptions{
 			AdoptExistingResources: opts.AdoptExistingResources,
@@ -573,9 +586,10 @@ func (d *deployer) installManifests(ctx context.Context, pkgLayout *layout.Packa
 		// Install the chart.
 		connectStrings, installedChartName, err := helm.InstallOrUpgradeChart(ctx, chart, helmChart, nil, helmOpts)
 		if err != nil {
-			return nil, err
+			installedCharts = append(installedCharts, state.InstalledChart{Namespace: manifest.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings, Status: state.ChartStatusFailed})
+			return installedCharts, err
 		}
-		installedCharts = append(installedCharts, state.InstalledChart{Namespace: manifest.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings})
+		installedCharts = append(installedCharts, state.InstalledChart{Namespace: manifest.Namespace, ChartName: installedChartName, ConnectStrings: connectStrings, Status: state.ChartStatusSucceeded})
 	}
 
 	return installedCharts, nil

src/pkg/packager/load/import.go

@@ -381,6 +381,12 @@ func overrideResources(comp v1alpha1.ZarfComponent, override v1alpha1.ZarfCompon
 				if overrideChart.ReleaseName != "" {
 					comp.Charts[idx].ReleaseName = overrideChart.ReleaseName
 				}
+				if overrideChart.Version != "" {
+					comp.Charts[idx].Version = overrideChart.Version
+				}
+				if overrideChart.URL != "" {
+					comp.Charts[idx].URL = overrideChart.URL
+				}
 				comp.Charts[idx].ValuesFiles = append(comp.Charts[idx].ValuesFiles, overrideChart.ValuesFiles...)
 				comp.Charts[idx].Variables = append(comp.Charts[idx].Variables, overrideChart.Variables...)
 				existing = true