Skip to content

Commit

Permalink
Add Ability to Store Export Data in AWS S3 Bucket (#45)
Browse files Browse the repository at this point in the history
* Allow override of image and version for exports
* Refactor export storage specification
* Add aws export example
* Add initial support for exports to AWS
* Add documentation for ArgoCDExport resource
  • Loading branch information
jomkz authored Apr 14, 2020
1 parent 9038f80 commit b2608a3
Show file tree
Hide file tree
Showing 23 changed files with 818 additions and 217 deletions.
3 changes: 3 additions & 0 deletions build/util/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM argoproj/argocd:v1.5.1

COPY util.sh /usr/local/bin/argocd-operator-util
114 changes: 114 additions & 0 deletions build/util/util.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
#!/bin/sh -e

# Copyright 2020 ArgoCD Operator Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

BACKUP_SCRIPT=$0
BACKUP_ACTION=$1
BACKUP_LOCATION=$2
BACKUP_FILENAME=argocd-backup.yaml
BACKUP_EXPORT_LOCATION=/tmp/${BACKUP_FILENAME}
BACKUP_ENCRYPT_LOCATION=/backups/${BACKUP_FILENAME}
BACKUP_KEY_LOCATION=/secrets/backup.key

export_argocd () {
echo "exporting argo-cd"
create_backup
encrypt_backup
push_backup
echo "argo-cd export complete"
}

create_backup () {
echo "creating argo-cd backup"
argocd-util export > ${BACKUP_EXPORT_LOCATION}
}

encrypt_backup () {
echo "encrypting argo-cd backup"
echo "backup hash: `md5sum ${BACKUP_KEY_LOCATION}`"
echo "export hash: `md5sum ${BACKUP_EXPORT_LOCATION}`"

openssl enc -aes-256-cbc -pbkdf2 -pass file:${BACKUP_KEY_LOCATION} -in ${BACKUP_EXPORT_LOCATION} -out ${BACKUP_ENCRYPT_LOCATION}
rm ${BACKUP_EXPORT_LOCATION}

echo "encrypt hash: `md5sum ${BACKUP_ENCRYPT_LOCATION}`"
}

push_backup () {
case ${BACKUP_LOCATION} in
"aws")
echo "pushing argo-cd backup to aws"
BACKUP_BUCKET_NAME=`cat /secrets/aws.bucket.name`
BACKUP_BUCKET_URI="s3://${BACKUP_BUCKET_NAME}"
aws s3 mb ${BACKUP_BUCKET_URI}
aws s3api put-public-access-block --bucket ${BACKUP_BUCKET_NAME} --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true"
aws s3 cp ${BACKUP_ENCRYPT_LOCATION} ${BACKUP_BUCKET_URI}/${BACKUP_FILENAME}
;;
*)
# local and unsupported backends
esac
}

import_argocd () {
echo "importing argo-cd"
pull_backup
decrypt_backup
load_backup
echo "argo-cd import complete"
}

pull_backup () {
case ${BACKUP_LOCATION} in
"aws")
echo "pulling argo-cd backup from aws"
BACKUP_BUCKET_NAME=`cat /secrets/aws.bucket.name`
BACKUP_BUCKET_URI="s3://${BACKUP_BUCKET_NAME}"
aws s3 cp ${BACKUP_BUCKET_URI}/${BACKUP_FILENAME} ${BACKUP_ENCRYPT_LOCATION}
;;
*)
# local and unsupported backends
esac
}

decrypt_backup () {
echo "decrypting argo-cd backup"
echo "backup hash: `md5sum ${BACKUP_KEY_LOCATION}`"
echo "encrypt hash: `md5sum ${BACKUP_ENCRYPT_LOCATION}`"

openssl enc -aes-256-cbc -d -pbkdf2 -pass file:${BACKUP_KEY_LOCATION} -in ${BACKUP_ENCRYPT_LOCATION} -out ${BACKUP_EXPORT_LOCATION}

echo "export hash: `md5sum ${BACKUP_EXPORT_LOCATION}`"
}

load_backup () {
echo "loading argo-cd backup"
argocd-util import - < ${BACKUP_EXPORT_LOCATION}
}

usage () {
echo "usage: ${BACKUP_SCRIPT} export|import"
}

case ${BACKUP_ACTION} in
"export")
export_argocd
;;
"import")
import_argocd
;;
# TODO: Implement finalize action to clean up cloud resources!
*)
usage
esac
232 changes: 120 additions & 112 deletions deploy/crds/argoproj.io_argocdexports_crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,134 +34,142 @@ spec:
argocd:
description: Argocd is the name of the ArgoCD instance to export.
type: string
image:
description: Image is the container image to use for the export Job.
type: string
schedule:
description: Schedule in Cron format, see https://en.wikipedia.org/wiki/Cron.
type: string
storage:
description: Storage defines the storage configuration options.
properties:
local:
description: Local defines options for storage local to the cluster.
backend:
description: Backend defines the storage backend to use, must be
one of "local" (the default) or "aws".
type: string
pvc:
description: PVC is the desired characteristics for a PersistentVolumeClaim.
properties:
pvc:
description: PVC is the desired characteristics for a PersistentVolumeClaim.
accessModes:
description: 'AccessModes contains the desired access modes
the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'
items:
type: string
type: array
dataSource:
description: This field requires the VolumeSnapshotDataSource
alpha feature gate to be enabled and currently VolumeSnapshot
is the only supported data source. If the provisioner can
support VolumeSnapshot data source, it will create a new volume
and data will be restored to the volume at the same time.
If the provisioner does not support VolumeSnapshot data source,
volume will not be created and the failure will be reported
as an event. In the future, we plan to support more data source
types and the behavior of the provisioner may change.
properties:
accessModes:
description: 'AccessModes contains the desired access modes
the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1'
items:
apiGroup:
description: APIGroup is the group for the resource being
referenced. If APIGroup is not specified, the specified
Kind must be in the core API group. For any other third-party
types, APIGroup is required.
type: string
kind:
description: Kind is the type of resource being referenced
type: string
name:
description: Name is the name of resource being referenced
type: string
required:
- kind
- name
type: object
resources:
description: 'Resources represents the minimum resources the
volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
properties:
limits:
additionalProperties:
type: string
type: array
dataSource:
description: This field requires the VolumeSnapshotDataSource
alpha feature gate to be enabled and currently VolumeSnapshot
is the only supported data source. If the provisioner
can support VolumeSnapshot data source, it will create
a new volume and data will be restored to the volume at
the same time. If the provisioner does not support VolumeSnapshot
data source, volume will not be created and the failure
will be reported as an event. In the future, we plan to
support more data source types and the behavior of the
provisioner may change.
properties:
apiGroup:
description: APIGroup is the group for the resource
being referenced. If APIGroup is not specified, the
specified Kind must be in the core API group. For
any other third-party types, APIGroup is required.
type: string
kind:
description: Kind is the type of resource being referenced
type: string
name:
description: Name is the name of resource being referenced
type: string
required:
- kind
- name
description: 'Limits describes the maximum amount of compute
resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
type: object
resources:
description: 'Resources represents the minimum resources
the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
properties:
limits:
additionalProperties:
type: string
description: 'Limits describes the maximum amount of
compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
type: object
requests:
additionalProperties:
type: string
description: 'Requests describes the minimum amount
of compute resources required. If Requests is omitted
for a container, it defaults to Limits if that is
explicitly specified, otherwise to an implementation-defined
value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
type: object
requests:
additionalProperties:
type: string
description: 'Requests describes the minimum amount of compute
resources required. If Requests is omitted for a container,
it defaults to Limits if that is explicitly specified,
otherwise to an implementation-defined value. More info:
https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
type: object
selector:
description: A label query over volumes to consider for
binding.
properties:
matchExpressions:
description: matchExpressions is a list of label selector
requirements. The requirements are ANDed.
items:
description: A label selector requirement is a selector
that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector
applies to.
type: string
operator:
description: operator represents a key's relationship
to a set of values. Valid operators are In,
NotIn, Exists and DoesNotExist.
type: string
values:
description: values is an array of string values.
If the operator is In or NotIn, the values array
must be non-empty. If the operator is Exists
or DoesNotExist, the values array must be empty.
This array is replaced during a strategic merge
patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: object
selector:
description: A label query over volumes to consider for binding.
properties:
matchExpressions:
description: matchExpressions is a list of label selector
requirements. The requirements are ANDed.
items:
description: A label selector requirement is a selector
that contains values, a key, and an operator that relates
the key and values.
properties:
key:
description: key is the label key that the selector
applies to.
type: string
description: matchLabels is a map of {key,value} pairs.
A single {key,value} in the matchLabels map is equivalent
to an element of matchExpressions, whose key field
is "key", the operator is "In", and the values array
contains only "value". The requirements are ANDed.
type: object
operator:
description: operator represents a key's relationship
to a set of values. Valid operators are In, NotIn,
Exists and DoesNotExist.
type: string
values:
description: values is an array of string values.
If the operator is In or NotIn, the values array
must be non-empty. If the operator is Exists or
DoesNotExist, the values array must be empty. This
array is replaced during a strategic merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs.
A single {key,value} in the matchLabels map is equivalent
to an element of matchExpressions, whose key field is
"key", the operator is "In", and the values array contains
only "value". The requirements are ANDed.
type: object
storageClassName:
description: 'Name of the StorageClass required by the claim.
More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'
type: string
volumeMode:
description: volumeMode defines what type of volume is required
by the claim. Value of Filesystem is implied when not
included in claim spec. This is a beta feature.
type: string
volumeName:
description: VolumeName is the binding reference to the
PersistentVolume backing this claim.
type: string
type: object
storageClassName:
description: 'Name of the StorageClass required by the claim.
More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1'
type: string
volumeMode:
description: volumeMode defines what type of volume is required
by the claim. Value of Filesystem is implied when not included
in claim spec. This is a beta feature.
type: string
volumeName:
description: VolumeName is the binding reference to the PersistentVolume
backing this claim.
type: string
type: object
secretName:
description: SecretName is the name of a Secret with encryption
key, credentials, etc.
type: string
type: object
version:
description: Version is the tag/digest to use for the export Job container
image.
type: string
required:
- argocd
type: object
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/argocd.md
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ kind: ArgoCD
metadata:
name: example-argocd
labels:
example: help-chat-text
example: image
spec:
image: argoproj/argocd
```
Expand Down
Loading

0 comments on commit b2608a3

Please sign in to comment.