refs sparkfabrik-innovation-team/board#715: add S3 ACL management (#16)

Author: Monska85

app/commands/bucket/main.sh

@@ -14,6 +14,7 @@ export FILE_SRC=${FILE_SRC:-""}
 export BUCKET_DST_ENDPOINT=${BUCKET_DST_ENDPOINT:-}
 export BUCKET_DST=${BUCKET_DST:-}
 export FILE_DST=${FILE_DST:-""}
+export ACL=${ACL:-}
 export RCLONE_ADD_PARAMS=${RCLONE_ADD_PARAMS:-}
 
 export TIMEOUT_BUCKET_SRC=${TIMEOUT_BUCKET_SRC:-10}
@@ -35,6 +36,7 @@ OPTIONS
   --bucket-dst-endpoint               Defines the destination bucket endpoint
   --bucket-dst                        Defines the destination bucket
   --file-dst                          Defines the destination file in the bucket
+  --acl                               Difines the ACL used to create the copied objects in the destination
   --rclone-add-params                 Defines the additional parameters to be passed to rclone command
   --timeout-bucket-src                Defines the maximum waiting time for source bucket set up (default ${TIMEOUT_BUCKET_SRC}s)
   --timeout-bucket-dst                Defines the maximum waiting time for destination bucket set up (default ${TIMEOUT_BUCKET_DST}s)
@@ -56,6 +58,7 @@ EOM
   printf "%-${PAD}s %s\n" "BUCKET_DST_ENDPOINT" "${BUCKET_DST_ENDPOINT}"
   printf "%-${PAD}s %s\n" "BUCKET_DST" "${BUCKET_DST}"
   printf "%-${PAD}s %s\n" "FILE_DST" "${FILE_DST}"
+  printf "%-${PAD}s %s\n" "ACL" "${ACL}"
   printf "%-${PAD}s %s\n" "RCLONE_ADD_PARAMS" "${RCLONE_ADD_PARAMS}"
   printf "%-${PAD}s %s\n" "TIMEOUT_BUCKET_SRC" "${TIMEOUT_BUCKET_SRC}"
   printf "%-${PAD}s %s\n" "TIMEOUT_BUCKET_DST" "${TIMEOUT_BUCKET_DST}"
@@ -82,6 +85,7 @@ while [ -n "${1:-}" ]; do
     --bucket-dst-endpoint) BUCKET_DST_ENDPOINT="${2}"; shift 2 ;;
     --bucket-dst) BUCKET_DST="${2}"; shift 2 ;;
     --file-dst) FILE_DST="${2}"; shift 2 ;;
+    --acl) ACL="${2}"; shift 2 ;;
     --rclone-add-params) RCLONE_ADD_PARAMS="${2}"; shift 2 ;;
     --timeout-bucket-src) TIMEOUT_BUCKET_SRC="${2}"; shift 2 ;;
     --timeout-bucket-dst) TIMEOUT_BUCKET_DST="${2}"; shift 2 ;;

app/commands/bucket/subcommands/copy-bucket.sh

@@ -44,6 +44,7 @@ elif [ -z "${FILE_SRC}" ]; then
 elif [ -z "${FILE_DST}" ]; then
   echo "$(format_string "Full bucket copy:" "bold") ${BUCKET_SRC}/${FILE_SRC} => ${BUCKET_DST}"
 fi
+echo "$(format_string "ACL:" "bold") ${ACL}"
 
 if [ "${PROVIDER_LOWER}" = "aws" ]; then
   echo "rclone_aws sync :s3://${BUCKET_SRC}/${FILE_SRC} :s3://${BUCKET_DST}/${FILE_DST}"

app/functions

@@ -32,31 +32,40 @@ format_string() {
   fi
 }
 
+S3_ACL=""
+if [ -n "${ACL}" ]; then
+  S3_ACL="--s3-acl ${ACL}"
+fi
+
 rclone_aws() {
+  echo "rclone --s3-region="${AWS_DEFAULT_REGION}" --s3-location-constraint="${AWS_DEFAULT_REGION}" --s3-env-auth ${S3_ACL} ${RCLONE_ADD_PARAMS} $@"
   rclone \
     --s3-region="${AWS_DEFAULT_REGION}" \
     --s3-location-constraint="${AWS_DEFAULT_REGION}" \
     --s3-env-auth \
-    ${RCLONE_ADD_PARAMS} $@
+    ${S3_ACL} ${RCLONE_ADD_PARAMS} $@
 }
 
 rclone_gcs() {
+  echo "rclone ${RCLONE_ADD_PARAMS} $@"
   rclone \
     ${RCLONE_ADD_PARAMS} $@
 }
 
 rclone_minio() {
+  echo "rclone --s3-provider=Minio --s3-endpoint=${BUCKET_ENDPOINT} --s3-env-auth ${S3_ACL} ${RCLONE_ADD_PARAMS} $@"
   rclone \
     --s3-provider=Minio \
     --s3-endpoint=${BUCKET_ENDPOINT} \
     --s3-env-auth \
-    ${RCLONE_ADD_PARAMS} $@
+    ${S3_ACL} ${RCLONE_ADD_PARAMS} $@
 }
 
 rclone_minio_multi() {
+  echo "rclone --config=/tmp/rclone.conf ${S3_ACL} ${RCLONE_ADD_PARAMS} $@"
   envsubst < ${BASE}/rclone.conf > /tmp/rclone.conf
   rclone --config=/tmp/rclone.conf \
-    ${RCLONE_ADD_PARAMS} $@
+    ${S3_ACL} ${RCLONE_ADD_PARAMS} $@
 }
 
 remove_http_proto() {

tests/functions

@@ -83,7 +83,7 @@ copyfiles() {
   FILE_DST="${2}"
 
   docker-compose -f "${COMPOSE_FILE}" run --rm ops-utils \
-    bucket copy-bucket --provider minio \
+    bucket copy-bucket --provider minio --acl public-read \
     --bucket-src-endpoint "http://minio:9000" --bucket-src seeds --file-src "${FILE_SRC}" \
     --bucket-dst-endpoint "http://minio-dst:9000" --bucket-dst dstbucket --file-dst "${FILE_DST}"
 }
@@ -97,7 +97,7 @@ copytobucket() {
   FILE_DST="${1}"
 
   docker-compose -f "${COMPOSE_FILE}" run --rm ops-utils \
-    bucket copy-bucket --provider minio \
+    bucket copy-bucket --provider minio --acl public-read \
     --bucket-src-endpoint "http://minio:9000" --bucket-src seeds \
     --bucket-dst-endpoint "http://minio-dst:9000" --bucket-dst dstbucket --file-dst "${FILE_DST}"
 }
@@ -111,7 +111,7 @@ copyfrombucket() {
   FILE_SRC="${1}"
 
   docker-compose -f "${COMPOSE_FILE}" run --rm ops-utils \
-    bucket copy-bucket --provider minio \
+    bucket copy-bucket --provider minio --acl public-read \
     --bucket-src-endpoint "http://minio:9000" --bucket-src seeds --file-src "${FILE_SRC}" \
     --bucket-dst-endpoint "http://minio-dst:9000" --bucket-dst dstbucket
 }