diff --git a/Makefile b/Makefile index 6a33bd8a80..e5d67887bd 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,7 @@ schema-check: # Is the generated Go code synced with the schema? grep -q "$(DIGEST)" pkg/builds/cosa_v1.go grep -q "$(DIGEST)" pkg/builds/schema_doc.go + grep -q "$(DIGEST)" src/cmd-cloud-prune install: install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler diff --git a/cmd/coreos-assembler.go b/cmd/coreos-assembler.go index 2824864fa5..696ac0ba02 100644 --- a/cmd/coreos-assembler.go +++ b/cmd/coreos-assembler.go @@ -16,7 +16,7 @@ var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean", var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container"} var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hashlist-experimental", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"} -var utilityCommands = []string{"aws-replicate", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-prune", "remote-session", "sign", "tag", "update-variant"} +var utilityCommands = []string{"aws-replicate", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "cloud-prune", "remote-session", "sign", "tag", "update-variant"} var otherCommands = []string{"shell", "meta"} func init() { diff --git a/mantle/cmd/ore/aws/delete-image.go b/mantle/cmd/ore/aws/delete-image.go new file mode 100644 index 0000000000..2ddb1b1092 --- /dev/null +++ b/mantle/cmd/ore/aws/delete-image.go @@ -0,0 +1,68 @@ +// Copyright 2017 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var ( + cmdDeleteImage = &cobra.Command{ + Use: "delete-image --ami --snapshot ...", + Short: "Delete AMI and/or snapshot", + Run: runDeleteImage, + } + amiID string + snapshotID string + allowMissing bool +) + +func init() { + // Initialize the command and its flags + AWS.AddCommand(cmdDeleteImage) + cmdDeleteImage.Flags().StringVar(&amiID, "ami", "", "AWS ami tag") + cmdDeleteImage.Flags().StringVar(&snapshotID, "snapshot", "", "AWS snapshot tag") + cmdDeleteImage.Flags().BoolVar(&allowMissing, "allow-missing", false, "Do not error out on the resource not existing") +} + +func runDeleteImage(cmd *cobra.Command, args []string) { + // Check if either amiID or snapshotID is provided + if amiID == "" && snapshotID == "" { + fmt.Fprintf(os.Stderr, "Provide --ami or --snapshot to delete\n") + os.Exit(1) + } + + // Remove resources based on provided flags + if amiID != "" { + err := API.RemoveByAmiTag(amiID, allowMissing) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not delete %v: %v\n", amiID, err) + os.Exit(1) + } + } + + if snapshotID != "" { + err := API.RemoveBySnapshotTag(snapshotID, allowMissing) + if err != nil { + fmt.Fprintf(os.Stderr, "Could not delete %v: %v\n", snapshotID, err) + os.Exit(1) + } + } + + os.Exit(0) +} diff --git a/mantle/cmd/ore/gcloud/delete-images.go b/mantle/cmd/ore/gcloud/delete-images.go index fe2f1773c5..3bcd062285 100644 --- a/mantle/cmd/ore/gcloud/delete-images.go +++ b/mantle/cmd/ore/gcloud/delete-images.go @@ -19,6 +19,7 @@ import ( "os" "github.com/spf13/cobra" + "google.golang.org/api/googleapi" "github.com/coreos/coreos-assembler/mantle/platform/api/gcloud" ) @@ -29,10 +30,12 @@ var ( Short: "Delete GCP images", Run: runDeleteImage, } + allowMissing bool ) func init() { GCloud.AddCommand(cmdDeleteImage) + cmdDeleteImage.Flags().BoolVar(&allowMissing, "allow-missing", false, "Do not error out on the resource not existing") } func runDeleteImage(cmd *cobra.Command, args []string) { @@ -46,7 +49,14 @@ func runDeleteImage(cmd *cobra.Command, args []string) { for _, name := range args { pending, err := api.DeleteImage(name) if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + if gErr, ok := err.(*googleapi.Error); ok { + // Skip on NotFound error only if allowMissing flag is set to True + if gErr.Code == 404 && allowMissing { + plog.Infof("%v\n", err) + continue + } + } + fmt.Fprintf(os.Stderr, "Deleting %q failed: %v\n", name, err) exit = 1 continue } diff --git a/mantle/platform/api/aws/images.go b/mantle/platform/api/aws/images.go index 70779a4830..bc4444b461 100644 --- a/mantle/platform/api/aws/images.go +++ b/mantle/platform/api/aws/images.go @@ -733,6 +733,49 @@ func (a *API) FindImage(name string) (string, error) { return "", nil } +// Deregisters the ami. +func (a *API) RemoveByAmiTag(imageID string, allowMissing bool) error { + _, err := a.ec2.DeregisterImage(&ec2.DeregisterImageInput{ImageId: &imageID}) + if err != nil { + if allowMissing { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "InvalidAMIID.NotFound" { + plog.Infof("%s does not exist.", imageID) + return nil + } + if awsErr.Code() == "InvalidAMIID.Unavailable" { + plog.Infof("%s is no longer available.", imageID) + return nil + } + } + } + return err + } + plog.Infof("Deregistered existing AMI %s", imageID) + return nil +} + +func (a *API) RemoveBySnapshotTag(snapshotID string, allowMissing bool) error { + _, err := a.ec2.DeleteSnapshot(&ec2.DeleteSnapshotInput{SnapshotId: &snapshotID}) + if err != nil { + if allowMissing { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "InvalidSnapshot.NotFound" { + plog.Infof("%s does not exist.", snapshotID) + return nil + } + if awsErr.Code() == "InvalidSnapshot.Unavailable" { + plog.Infof("%s is no longer available.", snapshotID) + return nil + } + } + } + return err + } + plog.Infof("Deregistered existing snapshot %s", snapshotID) + return nil +} + func (a *API) describeImage(imageID string) (*ec2.Image, error) { describeRes, err := a.ec2.DescribeImages(&ec2.DescribeImagesInput{ ImageIds: aws.StringSlice([]string{imageID}), diff --git a/mantle/platform/api/gcloud/image.go b/mantle/platform/api/gcloud/image.go index 715da0de39..e60d6a9944 100644 --- a/mantle/platform/api/gcloud/image.go +++ b/mantle/platform/api/gcloud/image.go @@ -223,7 +223,7 @@ func (a *API) DeprecateImage(name string, state DeprecationState, replacement st func (a *API) DeleteImage(name string) (*Pending, error) { op, err := a.compute.Images.Delete(a.options.Project, name).Do() if err != nil { - return nil, fmt.Errorf("Deleting %s failed: %v", name, err) + return nil, err } opReq := a.compute.GlobalOperations.Get(a.options.Project, op.Name) return a.NewPending(op.Name, opReq), nil diff --git a/src/cmd-buildupload b/src/cmd-buildupload index c278211a72..eef319c834 100755 --- a/src/cmd-buildupload +++ b/src/cmd-buildupload @@ -10,8 +10,7 @@ import sys import tempfile import subprocess import boto3 -from botocore.exceptions import ClientError, NoCredentialsError -from tenacity import retry +from cosalib.s3 import s3_copy, s3_check_exists sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -22,13 +21,7 @@ CACHE_MAX_AGE_ARTIFACT = 60 * 60 * 24 * 365 # set metadata caching to 5m CACHE_MAX_AGE_METADATA = 60 * 5 from cosalib.builds import Builds, BUILDFILES -from cosalib.cmdlib import ( - load_json, - retry_stop_long, - retry_wait_long, - retry_boto_exception, - retry_callback -) +from cosalib.cmdlib import load_json def main(): @@ -194,57 +187,5 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix): dry_run=args.dry_run) -@retry(stop=retry_stop_long, wait=retry_wait_long, - retry=retry_boto_exception, before_sleep=retry_callback) -def s3_check_exists(s3_client, bucket, key, dry_run=False): - print(f"Checking if bucket '{bucket}' has key '{key}'") - try: - s3_client.head_object(Bucket=bucket, Key=key) - except ClientError as e: - if e.response['Error']['Code'] == '404': - return False - raise e - except NoCredentialsError as e: - # It's reasonable to run without creds if doing a dry-run - if dry_run: - return False - raise e - return True - - -@retry(stop=retry_stop_long, wait=retry_wait_long, - retry=retry_boto_exception, retry_error_callback=retry_callback) -def s3_copy(s3_client, src, bucket, key, max_age, acl, extra_args={}, dry_run=False): - extra_args = dict(extra_args) - if 'ContentType' not in extra_args: - if key.endswith('.json'): - extra_args['ContentType'] = 'application/json' - elif key.endswith('.tar'): - extra_args['ContentType'] = 'application/x-tar' - elif key.endswith('.xz'): - extra_args['ContentType'] = 'application/x-xz' - elif key.endswith('.gz'): - extra_args['ContentType'] = 'application/gzip' - elif key.endswith('.iso'): - extra_args['ContentType'] = 'application/x-iso9660-image' - else: - # use a standard MIME type for "binary blob" instead of the default - # 'binary/octet-stream' AWS slaps on - extra_args['ContentType'] = 'application/octet-stream' - upload_args = { - 'CacheControl': f'max-age={max_age}', - 'ACL': acl - } - upload_args.update(extra_args) - - print((f"{'Would upload' if dry_run else 'Uploading'} {src} to " - f"s3://{bucket}/{key} with args {upload_args}")) - - if dry_run: - return - - s3_client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=upload_args) - - if __name__ == '__main__': sys.exit(main()) diff --git a/src/cmd-cloud-prune b/src/cmd-cloud-prune new file mode 100755 index 0000000000..2a1b3667c1 --- /dev/null +++ b/src/cmd-cloud-prune @@ -0,0 +1,297 @@ +#!/usr/bin/python3 -u + +# This script parses a policy.yaml file, which outlines the specific +# pruning actions required for each stream and the age threshold for +# deleting artifacts within them. +# Example of policy.yaml +# rawhide: +# # all cloud images +# cloud-uploads: 2 years +# # artifacts in meta.json's `images` key +# images: 2 years +# images-keep: [qemu, live-iso] +# build: 3 years +# The script also updates the builds.json for the respective stream by +# adding the policy-cleanup key when we set the upload_builds_json flag. +# It adds the relevant actions completed to that key +# For eg: +# "builds": [ +# { +# "id": "40.20240425.dev.1", +# "arches": [ +# "x86_64" +# ], +# "policy-cleanup": [ +# "cloud-uploads", +# "images-kept": ["qemu", "live-iso"] +# ] +# } + +import argparse +import json +from urllib.parse import urlparse +import pytz +import yaml +import collections +import datetime +import os +import boto3 +from dateutil.relativedelta import relativedelta +from cosalib.gcp import remove_gcp_image +from cosalib.aws import deregister_aws_resource +from cosalib.builds import BUILDFILES +from cosalib.s3 import s3_copy +from cosalib.cmdlib import parse_fcos_version_to_timestamp + +Build = collections.namedtuple("Build", ["id", "images", "arch", "meta_json"]) +# set metadata caching to 5m +CACHE_MAX_AGE_METADATA = 60 * 5 + + +def parse_args(): + parser = argparse.ArgumentParser(prog="coreos-assembler cloud-prune") + parser.add_argument("--policy", required=True, type=str, help="Path to policy YAML file") + parser.add_argument("--dry-run", help="Don't actually delete anything", action='store_true') + parser.add_argument("--upload-builds-json", help="Push builds.json", action='store_true') + parser.add_argument("--stream", type=str, help="CoreOS stream", required=True) + parser.add_argument("--gcp-json-key", help="GCP Service Account JSON Auth", default=os.environ.get("GCP_JSON_AUTH")) + parser.add_argument("--gcp-project", help="GCP Project name", default=os.environ.get("GCP_PROJECT_NAME")) + parser.add_argument("--acl", help="ACL for objects", action='store', default='private') + parser.add_argument("--aws-config-file", default=os.environ.get("AWS_CONFIG_FILE"), help="Path to AWS config file") + return parser.parse_args() + + +def main(): + # Parse arguments and initialize variables + args = parse_args() + with open(BUILDFILES['sourceurl'], "r") as file: + builds_source_data_url = file.read() + bucket, prefix = get_s3_bucket_and_prefix(builds_source_data_url) + cloud_config = get_cloud_config(args) + stream = args.stream + today_date = datetime.datetime.now() + + # Boto3 loads credentials from ~/.aws/config by default and we can change + # this default location by setting the AWS_CONFIG_FILE environment variable. + # The Python bindings don't support passing a config file. + # The alternative is to manually pass ACCESS_KEY and SECRET_KEY which isn't favourable. + if args.aws_config_file: + os.environ["AWS_CONFIG_FILE"] = args.aws_config_file + s3_client = boto3.client("s3") + + # Upload builds.json to s3 bucket + if args.upload_builds_json: + # This copies the local builds.json and updates the S3 bucket version. + return handle_upload_builds_json(s3_client, bucket, prefix, args.dry_run, args.acl) + + # These lists are up to date as of schema hash + # 4c19aed3b3d84af278780bff63728510bb3e70613e4c4eef8cabd7939eb31bd8. If changing + # this hash, ensure that the list of supported and unsupported artifacts below + # is up to date. + supported = ["amis", "gcp"] + unsupported = ["aliyun", "azurestack", "digitalocean", "exoscale", "ibmcloud", "powervs", "azure"] + + with open(args.policy, "r") as f: + policy = yaml.safe_load(f) + validate_policy(stream, policy) + + with open(BUILDFILES['list'], "r") as f: + builds_json_data = json.load(f) + + # Prune builds based on the policy + for action in ['cloud-uploads', 'images', 'build']: + if action not in policy[stream]: + continue + duration = get_period_in_months(policy[stream][action]) + ref_date = today_date - relativedelta(months=int(duration)) + + print(f"Pruning resources of type {action} older than {duration} months ({ref_date.date()}) on stream {stream}") + # Enumerating in reverse to go from the oldest build to the newest one + for index, build in enumerate(reversed(builds_json_data["builds"])): + build_id = build["id"] + if action in build.get("policy-cleanup", []): + print(f"Build {build_id} has already had {action} pruning completed") + continue + build_date = parse_fcos_version_to_timestamp(build_id) + + if build_date >= ref_date: + break + for arch in build["arches"]: + meta_prefix = os.path.join(prefix, f"{build_id}/{arch}/meta.json") + meta_json = get_json_from_s3(s3_client, bucket, meta_prefix) + # Make sure the meta.json doesn't contain any cloud_platform that is not supported for pruning yet. + images = get_supported_images(meta_json, unsupported, supported) + current_build = Build(id=build_id, images=images, arch=arch, meta_json=meta_json) + + match action: + case "cloud-uploads": + prune_cloud_uploads(current_build, cloud_config, args.dry_run) + case "build": + raise NotImplementedError + # print(f"Deleting key {prefix}{build.id} from bucket {bucket}") + # Delete the build's directory in S3 + # S3().delete_object(args.bucket, f"{args.prefix}{str(current_build.id)}") + case "images": + raise NotImplementedError + if not args.dry_run: + build.setdefault("policy-cleanup", []).append("cloud-uploads") + builds_json_data["builds"][index] = build + + if not args.dry_run: + builds_json_data["timestamp"] = datetime.datetime.now(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + # Save the updated builds.json to local builds/builds.json + save_builds_json(builds_json_data) + + +def get_s3_bucket_and_prefix(builds_source_data_url): + parsed_url = urlparse(builds_source_data_url) + if parsed_url.scheme == "s3": + bucket, prefix = parsed_url.netloc, parsed_url.path.lstrip("/") + return bucket, prefix + raise Exception("Invalid scheme: only s3:// supported") + + +def get_cloud_config(args): + return { + "gcp": { + "json-key": args.gcp_json_key, + "project": args.gcp_project + }, + "aws": { + "credentials": args.aws_config_file + } + } + + +def validate_policy(stream, policy): + # If the build key is set in the policy file, then the cloud-uploads key must + # also be present, and the duration of cloud-uploads must be equal or shorter + if "build" in policy[stream]: + actions = policy[stream] + if 'cloud-uploads' not in actions: + raise Exception("Pruning for cloud-uploads must be set before we prune the builds") + cloud_uploads_duration = get_period_in_months(actions["cloud-uploads"]) + build_duration = get_period_in_months(actions["build"]) + if cloud_uploads_duration > build_duration: + raise Exception("Duration of pruning cloud-uploads must be less than or equal to pruning a build") + + +def get_supported_images(meta_json, unsupported, supported): + images = {} + for key in meta_json: + if key in unsupported: + raise Exception(f"The platform {key} is not supported") + if key in supported: + images[key] = meta_json[key] + return images + + +def get_json_from_s3(s3, bucket, key): + try: + response = s3.get_object(Bucket=bucket, Key=key) + content = response["Body"].read().decode("utf-8") + return json.loads(content) + except Exception as e: + raise Exception(f"Error fetching the JSON file from S3 {bucket}/{key}: {e}") + + +def save_builds_json(builds_json_data): + with open(BUILDFILES['list'], "w") as json_file: + json.dump(builds_json_data, json_file, indent=2) + + +def handle_upload_builds_json(s3_client, bucket, prefix, dry_run, acl): + remote_builds_json = get_json_from_s3(s3_client, bucket, os.path.join(prefix, "builds.json")) + with open(BUILDFILES['sourcedata'], "r") as f: + builds_json_source_data = json.load(f) + # Check if there are any changes that were made to remote(s3 version) builds.json + # while the pruning was in progress + if remote_builds_json != builds_json_source_data: + print("Detected remote updates to builds.json. Merging it to the local builds.json file") + with open(BUILDFILES['list'], "r") as f: + current_builds_json = json.load(f) + update_policy_cleanup(current_builds_json, remote_builds_json) + if not dry_run: + # Make sure we have the merged json as local builds/builds.json + save_builds_json(remote_builds_json) + # Upload the local builds.json to s3 + return s3_copy(s3_client, BUILDFILES['list'], bucket, f'{prefix}/builds.json', CACHE_MAX_AGE_METADATA, acl, extra_args={}, dry_run=dry_run) + + +# Function to update policy-cleanup keys into remote_builds +def update_policy_cleanup(current_builds, remote_builds): + current_builds_dict = {build['id']: build for build in current_builds['builds']} + for remote_build in remote_builds['builds']: + build_id = remote_build['id'] + if build_id in current_builds_dict: + current_build = current_builds_dict[build_id] + if 'policy-cleanup' in current_build: + remote_build['policy-cleanup'] = current_build['policy-cleanup'] + + +def prune_cloud_uploads(build, cloud_config, dry_run): + # Ensure AWS AMIs and GCP images are removed based on the configuration + errors = [] + errors.extend(deregister_aws_amis(build, cloud_config, dry_run)) + errors.extend(delete_gcp_image(build, cloud_config, dry_run)) + + if errors: + print(f"Found errors when removing cloud-uploads for {build.id}:") + for e in errors: + print(e) + raise Exception("Some errors were encountered") + + +def deregister_aws_amis(build, cloud_config, dry_run): + errors = [] + aws_credentials = cloud_config.get("aws", {}).get("credentials") + for ami in build.images.get("amis", []): + region_name = ami.get("name") + ami_id = ami.get("hvm") + snapshot_id = ami.get("snapshot") + if dry_run: + print(f"Would delete {ami_id} and {snapshot_id} for {build.id}") + continue + if ami_id and snapshot_id and region_name: + try: + deregister_aws_resource(ami_id, snapshot_id, region=region_name, credentials_file=aws_credentials) + except Exception as e: + errors.append(e) + else: + errors.append(f"Missing parameters to remove {ami_id} and {snapshot_id}") + return errors + + +def delete_gcp_image(build, cloud_config, dry_run): + errors = [] + gcp = build.images.get("gcp") + if not gcp: + print(f"No GCP image for {build.id} for {build.arch}") + return + gcp_image = gcp.get("image") + json_key = cloud_config.get("gcp", {}).get("json-key") + project = cloud_config.get("gcp", {}).get("project") + if dry_run: + print(f"Would delete {gcp_image} GCP image for {build.id}") + elif gcp_image and json_key and project: + try: + remove_gcp_image(gcp_image, json_key, project) + except Exception as e: + errors.append(e) + else: + errors.append(f"Missing parameters to remove {gcp_image}") + return errors + + +def get_period_in_months(duration): + val, unit = duration.split(maxsplit=1) + if unit in ["years", "year", "y"]: + return int(val) * 12 + elif unit in ["months", "month", "m"]: + return int(val) + else: + raise Exception(f"Duration unit provided is {unit}. Pruning duration is only supported in years and months") + + +if __name__ == "__main__": + main() diff --git a/src/cosalib/aws.py b/src/cosalib/aws.py index 12869d13d7..eab2aa6972 100644 --- a/src/cosalib/aws.py +++ b/src/cosalib/aws.py @@ -1,4 +1,3 @@ -import boto3 import json import os import subprocess @@ -6,9 +5,7 @@ from cosalib.cmdlib import ( flatten_image_yaml, - retry_boto_exception, - retry_callback, - retry_stop + runcmd ) from tenacity import ( retry, @@ -16,20 +13,21 @@ ) -@retry(stop=retry_stop, retry=retry_boto_exception, - before_sleep=retry_callback) -def deregister_ami(ami_id, region): - print(f"AWS: deregistering AMI {ami_id} in {region}") - ec2 = boto3.client('ec2', region_name=region) - ec2.deregister_image(ImageId=ami_id) - - -@retry(stop=retry_stop, retry=retry_boto_exception, - before_sleep=retry_callback) -def delete_snapshot(snap_id, region): - print(f"AWS: removing snapshot {snap_id} in {region}") - ec2 = boto3.client('ec2', region_name=region) - ec2.delete_snapshot(SnapshotId=snap_id) +@retry(reraise=True, stop=stop_after_attempt(3)) +def deregister_aws_resource(ami, snapshot, region, credentials_file): + print(f"AWS: deregistering AMI {ami} and {snapshot} in {region}") + try: + runcmd([ + 'ore', 'aws', 'delete-image', + '--credentials-file', credentials_file, + '--ami', ami, + '--snapshot', snapshot, + "--region", region, + "--allow-missing" + ]) + print(f"AWS: successfully removed {ami} and {snapshot}") + except SystemExit: + raise Exception(f"Failed to remove {ami} or {snapshot}") @retry(reraise=True, stop=stop_after_attempt(3)) diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index e21efc3808..613a80a927 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -7,6 +7,7 @@ import json import logging import os +import re import shutil import subprocess import sys @@ -338,6 +339,21 @@ def get_basearch(): return get_basearch.saved +def parse_fcos_version_to_timestamp(version): + ''' + Parses an FCOS build ID and verifies the versioning is accurate. Then + it verifies that the parsed timestamp has %Y%m%d format and returns that. + ''' + m = re.match(r'^([0-9]{2})\.([0-9]{8})\.([0-9]+)\.([0-9]+)$', version) + if m is None: + raise Exception(f"Incorrect versioning for FCOS build {version}") + try: + timestamp = datetime.datetime.strptime(m.group(2), '%Y%m%d') + except ValueError: + raise Exception(f"FCOS build {version} has incorrect date format. It should be in (%Y%m%d)") + return timestamp + + def parse_date_string(date_string): """ Parses the date strings expected from the build system. Returned diff --git a/src/cosalib/gcp.py b/src/cosalib/gcp.py index 39cdad4196..b333e9f65f 100644 --- a/src/cosalib/gcp.py +++ b/src/cosalib/gcp.py @@ -24,10 +24,12 @@ def remove_gcp_image(gcp_id, json_key, project): runcmd([ 'ore', 'gcloud', 'delete-images', gcp_id, '--json-key', json_key, - '--project', project + '--project', project, + '--allow-missing' ]) + print(f"GCP: successfully removed image {gcp_id}") except SystemExit: - raise Exception("Failed to remove image") + raise Exception(f"Failed to remove image {gcp_id}") @retry(reraise=True, stop=stop_after_attempt(3)) diff --git a/src/cosalib/prune.py b/src/cosalib/prune.py index eee6c338b3..553b8119ad 100644 --- a/src/cosalib/prune.py +++ b/src/cosalib/prune.py @@ -4,10 +4,7 @@ from cosalib.s3 import S3 -from cosalib.aws import ( - deregister_ami, - delete_snapshot -) +from cosalib.aws import deregister_aws_resource from cosalib.aliyun import remove_aliyun_image from cosalib.gcp import remove_gcp_image @@ -85,14 +82,10 @@ def delete_build(build, bucket, prefix, cloud_config, force=False): region_name = ami.get('name') ami_id = ami.get('hvm') snapshot_id = ami.get('snapshot') - if ami_id and region_name: + aws_credentials = cloud_config.get('aws', {}).get('credentials') + if snapshot_id and ami_id and region_name: try: - deregister_ami(ami_id, region=region_name) - except Exception as e: - errors.append(e) - if snapshot_id and region_name: - try: - delete_snapshot(snapshot_id, region=region_name) + deregister_aws_resource(ami_id, snapshot_id, region=region_name, credentials_file=aws_credentials) except Exception as e: errors.append(e) diff --git a/src/cosalib/s3.py b/src/cosalib/s3.py index 356e30121b..7066c0272c 100644 --- a/src/cosalib/s3.py +++ b/src/cosalib/s3.py @@ -1,8 +1,10 @@ import boto3 -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, NoCredentialsError from cosalib.cmdlib import ( retry_stop, + retry_stop_long, + retry_wait_long, retry_boto_exception, retry_callback ) @@ -52,3 +54,55 @@ def delete_object(self, bucket, key): print("S3: deleting {sub_objects}") self.client.delete_objects(Bucket=bucket, Delete=sub_objects) self.client.delete_object(Bucket=bucket, Key=key) + + +@retry(stop=retry_stop_long, wait=retry_wait_long, + retry=retry_boto_exception, before_sleep=retry_callback) +def s3_check_exists(s3_client, bucket, key, dry_run=False): + print(f"Checking if bucket '{bucket}' has key '{key}'") + try: + s3_client.head_object(Bucket=bucket, Key=key) + except ClientError as e: + if e.response['Error']['Code'] == '404': + return False + raise e + except NoCredentialsError as e: + # It's reasonable to run without creds if doing a dry-run + if dry_run: + return False + raise e + return True + + +@retry(stop=retry_stop_long, wait=retry_wait_long, + retry=retry_boto_exception, retry_error_callback=retry_callback) +def s3_copy(s3_client, src, bucket, key, max_age, acl, extra_args={}, dry_run=False): + extra_args = dict(extra_args) + if 'ContentType' not in extra_args: + if key.endswith('.json'): + extra_args['ContentType'] = 'application/json' + elif key.endswith('.tar'): + extra_args['ContentType'] = 'application/x-tar' + elif key.endswith('.xz'): + extra_args['ContentType'] = 'application/x-xz' + elif key.endswith('.gz'): + extra_args['ContentType'] = 'application/gzip' + elif key.endswith('.iso'): + extra_args['ContentType'] = 'application/x-iso9660-image' + else: + # use a standard MIME type for "binary blob" instead of the default + # 'binary/octet-stream' AWS slaps on + extra_args['ContentType'] = 'application/octet-stream' + upload_args = { + 'CacheControl': f'max-age={max_age}', + 'ACL': acl + } + upload_args.update(extra_args) + + print((f"{'Would upload' if dry_run else 'Uploading'} {src} to " + f"s3://{bucket}/{key} {extra_args if len(extra_args) else ''}")) + + if dry_run: + return + + s3_client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=upload_args)