refs sparkfabrik-innovation-team/board#807: add support for cross provider buckets

Author: Monska85

Makefile

@@ -17,6 +17,7 @@ cli-dev: build
 	docker run --rm -it \
 		--env-file .env \
 		-v ${PWD}/app:/app \
+		-v ${HOME}/.config/gcloud:/root/.config/gcloud \
 		-w /app \
 		$(IMAGE_NAME):$(IMAGE_TAG) ash -li
 

app/commands/bucket/main.sh

@@ -7,7 +7,10 @@ SCRIPT=$(basename $0)
 export SUBCOMMAND=""
 
 export PROVIDER=${PROVIDER:-}
-export PROVIDER_LOWER=
+export PROVIDER_SRC=${PROVIDER_SRC:-}
+export PROVIDER_SRC_LOWER=
+export PROVIDER_DST=${PROVIDER_DST:-}
+export PROVIDER_DST_LOWER=
 export BUCKET_SRC_ENDPOINT=${BUCKET_SRC_ENDPOINT:-}
 export BUCKET_SRC=${BUCKET_SRC:-}
 export FILE_SRC=${FILE_SRC:-""}
@@ -78,7 +81,9 @@ shift
 PARAMS=""
 while [ -n "${1:-}" ]; do
   case "${1}" in
-    --provider) PROVIDER="${2}"; shift 2 ;;
+    --provider) PROVIDER_SRC="${2}"; PROVIDER_DST="${2}"; shift 2 ;;
+    --provider-src) PROVIDER_SRC="${2}"; shift 2 ;;
+    --provider-dst) PROVIDER_DST="${2}"; shift 2 ;;
     --bucket-src-endpoint) BUCKET_SRC_ENDPOINT="${2}"; shift 2 ;;
     --bucket-src) BUCKET_SRC="${2}"; shift 2 ;;
     --file-src) FILE_SRC="${2}"; shift 2 ;;
@@ -96,7 +101,8 @@ done
 
 eval set -- "$PARAMS"
 
-PROVIDER_LOWER=$(echo ${PROVIDER} | awk '{print tolower($0)}')
+PROVIDER_SRC_LOWER=$(echo ${PROVIDER_SRC} | awk '{print tolower($0)}')
+PROVIDER_DST_LOWER=$(echo ${PROVIDER_DST} | awk '{print tolower($0)}')
 
 # Check dry run execution
 if [ ${DRY_RUN} -eq 1 ]; then

app/commands/bucket/subcommands/copy-bucket.sh

@@ -4,23 +4,31 @@
 . ${BASE}/functions
 
 # Check for the required input
-if [ -z "${PROVIDER}" ]; then
-  echo "You have to define the bucket provider"
+if [ -z "${PROVIDER_SRC}" ]; then
+  echo "You have to define the source bucket provider"
   exit 12
 fi
-if [ "${PROVIDER_LOWER}" != "aws" ] && [ "${PROVIDER_LOWER}" != "gcs" ] && [ "${PROVIDER_LOWER}" != "minio" ]; then
+if [ -z "${PROVIDER_DST}" ]; then
+  echo "You have to define the destination bucket provider"
+  exit 12
+fi
+if [ "${PROVIDER_SRC_LOWER}" != "aws" ] && [ "${PROVIDER_SRC_LOWER}" != "gcs" ] && [ "${PROVIDER_SRC_LOWER}" != "minio" ]; then
   echo "You have to define a valid bucket provider (aws, gcs, minio)"
   exit 12
 fi
-if [ "${PROVIDER_LOWER}" = "minio" ] && [ -z "${BUCKET_SRC_ENDPOINT}" ]; then
+if [ "${PROVIDER_DST_LOWER}" != "aws" ] && [ "${PROVIDER_DST_LOWER}" != "gcs" ] && [ "${PROVIDER_DST_LOWER}" != "minio" ]; then
+  echo "You have to define a valid bucket provider (aws, gcs, minio)"
+  exit 12
+fi
+if [ "${PROVIDER_SRC_LOWER}" = "minio" ] && [ -z "${BUCKET_SRC_ENDPOINT}" ]; then
   echo "You have to define the source bucket endpoint"
   exit 12
 fi
 if [ -z "${BUCKET_SRC}" ]; then
   echo "You have to define the source bucket name"
   exit 12
 fi
-if [ "${PROVIDER_LOWER}" = "minio" ] && [ -z "${BUCKET_DST_ENDPOINT}" ]; then
+if [ "${PROVIDER_DST_LOWER}" = "minio" ] && [ -z "${BUCKET_DST_ENDPOINT}" ]; then
   echo "You have to define the destination bucket endpoint"
   exit 12
 fi
@@ -34,7 +42,8 @@ echo "All the required inputs are present. Go on with the real job."
 
 echo "Copy the file from the source bucket to the destination bucket."
 format_string "Parameters:" "g"
-echo "$(format_string "Provider:" "bold") ${PROVIDER_LOWER}"
+echo "$(format_string "Src Provider:" "bold") ${PROVIDER_SRC_LOWER}"
+echo "$(format_string "Dst Provider:" "bold") ${PROVIDER_DST_LOWER}"
 echo "$(format_string "Src:" "bold") ${BUCKET_SRC}/${FILE_SRC}"
 echo "$(format_string "Dst:" "bold") ${BUCKET_DST}/${FILE_DST}"
 if [ -z "${FILE_SRC}" ] && [ -z "${FILE_DST}" ]; then
@@ -46,15 +55,7 @@ elif [ -z "${FILE_DST}" ]; then
 fi
 echo "$(format_string "ACL:" "bold") ${ACL}"
 
-if [ "${PROVIDER_LOWER}" = "aws" ]; then
-  echo "rclone_aws sync :s3://${BUCKET_SRC}/${FILE_SRC} :s3://${BUCKET_DST}/${FILE_DST}"
-  rclone_aws sync :s3://${BUCKET_SRC}/${FILE_SRC} :s3://${BUCKET_DST}/${FILE_DST}
-  EXIT_RCLONE=$?
-elif [ "${PROVIDER_LOWER}" = "gcs" ]; then
-  echo "rclone_gcs sync :gcs://${BUCKET_SRC}/${FILE_SRC} :gcs://${BUCKET_SRC}/${FILE_DST}"
-  rclone_gcs sync :gcs://${BUCKET_SRC}/${FILE_SRC} :gcs://${BUCKET_SRC}/${FILE_DST}
-  EXIT_RCLONE=$?
-elif [ "${PROVIDER_LOWER}" = "minio" ]; then
+if [ "${PROVIDER_LOWER}" = "minio" ]; then
   # Wait for source minio service
   WAIT_ENDPOINT=$(remove_http_proto "${BUCKET_SRC_ENDPOINT}")
   debug "Wait for source minio service (${WAIT_ENDPOINT}, timeout ${TIMEOUT_BUCKET_SRC} seconds)."
@@ -97,12 +98,12 @@ elif [ "${PROVIDER_LOWER}" = "minio" ]; then
       exit 13
     fi
   done
-
-  echo "rclone_minio_multi sync src://${BUCKET_SRC}/${FILE_SRC} dst://${BUCKET_DST}/${FILE_DST}"
-  rclone_minio_multi sync src://${BUCKET_SRC}/${FILE_SRC} dst://${BUCKET_DST}/${FILE_DST}
-  EXIT_RCLONE=$?
 fi
 
+echo "rclone_generic sync src://${BUCKET_SRC}/${FILE_SRC} dst://${BUCKET_DST}/${FILE_DST}"
+rclone_generic sync src://${BUCKET_SRC}/${FILE_SRC} dst://${BUCKET_DST}/${FILE_DST}
+EXIT_RCLONE=$?
+
 if [ ${EXIT_RCLONE} -ne 0 ]; then
   echo "Something went wrong during the copy of files."
   exit ${EXIT_RCLONE}

app/functions

@@ -37,6 +37,38 @@ if [ -n "${ACL}" ]; then
   S3_ACL="--s3-acl ${ACL}"
 fi
 
+generate_rclone_conf() {
+  rm -f /tmp/rclone.conf
+  echo "[src]" | tee -a /tmp/rclone.conf
+  if [ "${PROVIDER_SRC_LOWER}" = "aws" ]; then
+    envsubst '${AWS_DEFAULT_REGION}' < ${BASE}/rclone.aws.conf | tee -a /tmp/rclone.conf
+  elif [ "${PROVIDER_SRC_LOWER}" = "gcs" ]; then
+    cat ${BASE}/rclone.gcs.conf | tee -a /tmp/rclone.conf
+  elif [ "${PROVIDER_SRC_LOWER}" = "minio" ]; then
+    BUCKET_ENDPOINT=${BUCKET_SRC_ENDPOINT} envsubst '${BUCKET_ENDPOINT}' < ${BASE}/rclone.minio.conf | tee -a /tmp/rclone.conf
+  fi
+
+  echo "" | tee -a /tmp/rclone.conf
+
+  echo "[dst]" | tee -a /tmp/rclone.conf
+  if [ "${PROVIDER_DST_LOWER}" = "aws" ]; then
+    envsubst '${AWS_DEFAULT_REGION}' < ${BASE}/rclone.aws.conf | tee -a /tmp/rclone.conf
+  elif [ "${PROVIDER_DST_LOWER}" = "gcs" ]; then
+    cat ${BASE}/rclone.gcs.conf | tee -a /tmp/rclone.conf
+  elif [ "${PROVIDER_DST_LOWER}" = "minio" ]; then
+    BUCKET_ENDPOINT=${BUCKET_DST_ENDPOINT} envsubst '${BUCKET_ENDPOINT}' < ${BASE}/rclone.minio.conf | tee -a /tmp/rclone.conf
+  fi
+  
+  echo "" | tee -a /tmp/rclone.conf
+}
+
+rclone_generic() {
+  generate_rclone_conf
+  echo "rclone --config=/tmp/rclone.conf ${S3_ACL} ${RCLONE_ADD_PARAMS} $@"
+  rclone --config=/tmp/rclone.conf \
+    ${S3_ACL} ${RCLONE_ADD_PARAMS} $@
+}
+
 rclone_aws() {
   echo "rclone --s3-region="${AWS_DEFAULT_REGION}" --s3-location-constraint="${AWS_DEFAULT_REGION}" --s3-env-auth ${S3_ACL} ${RCLONE_ADD_PARAMS} $@"
   rclone \

app/rclone.aws.conf

@@ -0,0 +1,5 @@
+type = s3
+provider = aws
+env_auth = true
+region = ${AWS_DEFAULT_REGION}
+location_constraint = ${AWS_DEFAULT_REGION}

app/rclone.gcs.conf

@@ -0,0 +1,4 @@
+type = google cloud storage
+provider = gcs
+env_auth = true
+token = ~/.config/gcloud/application_default_credentials.json

app/rclone.minio.conf

@@ -0,0 +1,4 @@
+type = s3
+provider = Minio
+env_auth = true
+endpoint = ${BUCKET_ENDPOINT}