Merge upstream 2.6.0

.github/dependabot.yml

@@ -5,6 +5,10 @@
 
 version: 2
 updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
   - package-ecosystem: "maven" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:

.github/workflows/ci-main.yml

@@ -3,6 +3,8 @@ on:
   push:
     branches:
       - "master"
+    tags:
+      - "*"
   pull_request:
     branches:
       - "*"
@@ -13,7 +15,7 @@ env:
 jobs:
 
   meta:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04-arm
     outputs:
       container_tags: ${{ steps.docker_action_meta.outputs.tags }}
       container_labels: ${{ steps.docker_action_meta.outputs.labels }}
@@ -28,13 +30,13 @@ jobs:
           persist-credentials: false
       - name: Docker meta
         id: docker_action_meta
-        uses: docker/metadata-action@v4.0.1
+        uses: docker/metadata-action@v5.7.0
         with:
           images: |
             name=ghcr.io/${{ github.repository }}/container
             name=andrewgaul/s3proxy,enable=${{ env.dockerhub_publish }}
           flavor: |
-            latest=false
+            latest=auto
           tags: |
             type=sha,format=long
             type=sha
@@ -45,7 +47,7 @@ jobs:
           labels: |
             org.opencontainers.image.licenses=Apache-2.0
   runTests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04-arm
     needs: [meta]
     steps:
       - uses: actions/checkout@v4
@@ -57,9 +59,9 @@ jobs:
           distribution: "temurin"
           java-version: "11"
           cache: "maven"
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.11"
           cache: "pip"
 
       #Run tests
@@ -72,35 +74,85 @@ jobs:
       - name: Maven Test
         run: |
           mvn test
-      - name: Other Test
+
+      - name: Maven Test with transient-nio2
+        run: |
+          # TODO: run other test classes
+          mvn test -Ds3proxy.test.conf=s3proxy-transient-nio2.conf -Dtest=AwsSdkTest
+
+      - name: Maven Test with filesystem-nio2
+        run: |
+          # TODO: run other test classes
+          mkdir /tmp/blobstore
+          mvn test -Ds3proxy.test.conf=s3proxy-filesystem-nio2.conf -Dtest=AwsSdkTest
+
+      - name: Install Azurite
+        run: npx --yes --loglevel info azurite@3.34 --version
+      - name: Start Azurite
+        shell: bash
+        run: npx --yes azurite@3.34 azurite-blob &
+      - name: Maven Test with Azurite
+        run: |
+          # TODO: run other test classes
+          mvn test -Ds3proxy.test.conf=s3proxy-azurite.conf -Dtest=AwsSdkTest
+
+      - name: Install Minio
+        run: |
+          curl -o minio https://dl.min.io/server/minio/release/linux-$(uname -m | sed -e s/aarch64/arm64/ -e s/x86_64/amd64/)/minio
+          chmod +x minio
+      - name: Start Minio
+        run: |
+          mkdir mnt/
+          MINIO_SERVER_URL=http://127.0.0.1:9000 MINIO_ROOT_USER=remote-identity MINIO_ROOT_PASSWORD=remote-credential ./minio server mnt/ &
+      - name: Maven Test with Minio
+        run: |
+          # TODO: run other test classes
+          mvn test -Ds3proxy.test.conf=s3proxy-minio.conf -Dtest=AwsSdkTest
+
+      - name: Install s3-tests
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox tox-gh-actions
+      - name: Run s3-tests
         run: |
           ./src/test/resources/run-s3-tests.sh
+      - name: Run s3-tests with transient-nio2
+        run: |
+          ./src/test/resources/run-s3-tests.sh s3proxy-transient-nio2.conf
+      - name: Run s3-tests with Azurite
+        run: |
+          ./src/test/resources/run-s3-tests.sh s3proxy-azurite.conf
+          kill $(pidof node)
+      - name: Run s3-tests with Minio
+        run: |
+          ./src/test/resources/run-s3-tests.sh s3proxy-minio.conf
+          kill $(pidof minio)
 
       #Store the target
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: s3proxy
           path: target/s3proxy
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: pom
           path: pom.xml
 
   Containerize:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04-arm
     needs: [runTests, meta]
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         with:
           name: s3proxy
           path: target
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         with:
           name: pom
           path: .
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -120,7 +172,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm/v7

Dockerfile

@@ -3,12 +3,16 @@
 
 WORKDIR /opt/s3proxy
 
+RUN apt-get update && \
+    apt-get install -y dumb-init && \
+    rm -rf /var/lib/apt/lists/*
+
 COPY \
     target/s3proxy \
     src/main/resources/run-docker-container.sh \
     /opt/s3proxy/
 
 ENV \
     LOG_LEVEL="info" \
     S3PROXY_AUTHORIZATION="aws-v2-or-v4" \
     S3PROXY_ENDPOINT="http://0.0.0.0:80" \
@@ -27,7 +31,7 @@
     S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD="" \
     S3PROXY_ENCRYPTED_BLOBSTORE_SALT="" \
     S3PROXY_READ_ONLY_BLOBSTORE="false" \
-    JCLOUDS_PROVIDER="filesystem" \
+    JCLOUDS_PROVIDER="filesystem-nio2" \
     JCLOUDS_ENDPOINT="" \
     JCLOUDS_REGION="" \
     JCLOUDS_REGIONS="us-east-1" \
@@ -39,4 +43,7 @@
     JCLOUDS_FILESYSTEM_BASEDIR="/data"
 
 EXPOSE 80 443
-ENTRYPOINT ["/opt/s3proxy/run-docker-container.sh"]
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["/opt/s3proxy/run-docker-container.sh"]

README.md

@@ -74,13 +74,16 @@ Maven Central hosts S3Proxy artifacts and the wiki has
 * atmos
 * aws-s3 (Amazon-only)
 * azureblob
+* azureblob-sdk (newer but lacks multi-part upload, see [Azure/azure-sdk-for-java#42603](https://github.com/Azure/azure-sdk-for-java/issues/42603))
 * b2
-* filesystem (on-disk storage)
+* filesystem (on-disk storage, deprecated)
+* filesystem-nio2 (on-disk storage, recommended)
 * google-cloud-storage
 * openstack-swift
 * rackspace-cloudfiles-uk and rackspace-cloudfiles-us
 * s3 (all implementations)
-* transient (in-memory storage)
+* transient (in-memory storage, deprecated)
+* transient-nio2 (in-memory storage, recommended)
 
 See the wiki for [examples of configurations](https://github.com/gaul/s3proxy/wiki/Storage-backend-examples).
 
@@ -107,8 +110,10 @@ S3Proxy can modify its behavior based on middlewares:
 * [eventual consistency modeling](https://github.com/gaul/s3proxy/wiki/Middleware---eventual-consistency)
 * [large object mocking](https://github.com/gaul/s3proxy/wiki/Middleware-large-object-mocking)
 * [read-only](https://github.com/gaul/s3proxy/wiki/Middleware-read-only)
-* [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend)
 * [regex rename blobs](https://github.com/gaul/s3proxy/wiki/Middleware-regex)
+* [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend)
+* [storage class override](https://github.com/gaul/s3proxy/wiki/Middleware-storage-class-override)
+* [user metadata replacer](https://github.com/gaul/s3proxy/wiki/Middleware-user-metadata-replacer)
 
 ## SSL Support
 
@@ -164,13 +169,13 @@ for specific storage backends.
 * [Ceph s3-tests](https://github.com/ceph/s3-tests) help maintain and improve compatibility with the S3 API
 * [fake-s3](https://github.com/jubos/fake-s3), [gofakes3](https://github.com/johannesboyne/gofakes3), [minio](https://github.com/minio/minio), [S3 ninja](https://github.com/scireum/s3ninja), and [s3rver](https://github.com/jamhall/s3rver) provide functionality similar to S3Proxy when using the filesystem backend
 * [GlacierProxy](https://github.com/bouncestorage/glacier-proxy) and [SwiftProxy](https://github.com/bouncestorage/swiftproxy) provide similar functionality for the Amazon Glacier and OpenStack Swift APIs
-* [s3mock](https://github.com/findify/s3mock) mocks the S3 API for Java/Scala projects
+* [s3mock](https://github.com/adobe/S3Mock) - Adobe's s3 mock implementation
 * [sbt-s3](https://github.com/localytics/sbt-s3) runs S3Proxy via the Scala Build Tool
 * [swift3](https://github.com/openstack/swift3) provides an S3 middleware for OpenStack Swift
 * [Zenko](https://www.zenko.io/) provide similar multi-cloud functionality
 
 ## License
 
-Copyright (C) 2014-2021 Andrew Gaul
+Copyright (C) 2014-2025 Andrew Gaul
 
 Licensed under the Apache License, Version 2.0

docs/Logging.md

@@ -0,0 +1,8 @@
+# Logging
+
+## Configuration
+
+The following environment variables can be used to configure logging
+
+* LOG_LEVEL default value "info" used to configure log level
+* LOG_APPENDER default value "STDOUT" produce string formatted logs "CONTAINER" used to produce json formatted logs