Refs/heads/fix/53 pypi missing subpackages

.github/workflows/ci.yaml

@@ -14,6 +14,8 @@ name: CI
     tags:
       - "*"
   pull_request: {}
+  release:
+    types: [published]
 
 jobs:
   lint:
@@ -26,7 +28,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.1
@@ -87,7 +89,7 @@ jobs:
           fetch-depth: 0 # full history for setuptools_scm
 
       - name: Build and publish
-        uses: lsst-sqre/build-and-publish-to-pypi@v2
+        uses: lsst-sqre/build-and-publish-to-pypi@v3
         with:
           python-version: "3.12"
           upload: false
@@ -114,6 +116,6 @@ jobs:
           fetch-depth: 0 # full history for setuptools_scm
 
       - name: Build and publish
-        uses: lsst-sqre/build-and-publish-to-pypi@v2
+        uses: lsst-sqre/build-and-publish-to-pypi@v3
         with:
           python-version: "3.12"

CHANGELOG.md

@@ -0,0 +1,23 @@
+
+<a id='changelog-0.6.2'></a>
+## 0.6.2 (2024-12-16)
+
+### Bug fixes
+
+- Correct typo in pyproject.toml
+
+<a id='changelog-v6.0.1'></a>
+## v6.0.1 (2024-12-16)
+
+### Bug fixes
+
+- Make CI attempt to upload on release
+
+<a id='changelog-v0.6.0'></a>
+## v0.6.0 (2024-12-16)
+
+### New features
+
+- Support Python 3.13
+
+- Work through PyPi publication

Dockerfile

@@ -1,61 +1,99 @@
 # Dockerfile for uWSGI wrapped Giftless Git LFS Server
+# Shared build ARGs among stages
+ARG WORKDIR=/app
+ARG VENV="$WORKDIR/.venv"
+ARG UV_VERSION=0.5.16
 
-### --- Build Depdendencies ---
-
-FROM python:3.12 as builder
-MAINTAINER "Shahar Evron <shahar.evron@datopian.com>"
-
-# Build wheels for uWSGI and all requirements
-RUN DEBIAN_FRONTEND=noninteractive apt-get update \
-    && apt-get install -y build-essential libpcre3 libpcre3-dev git
-RUN pip install -U pip
-RUN mkdir /wheels
+### Distroless uv version layer to be copied from (because COPY --from does not interpolate variables)
+FROM ghcr.io/astral-sh/uv:$UV_VERSION AS uv
 
+### --- Build Depdendencies ---
+FROM python:3.12 AS builder
 ARG UWSGI_VERSION=2.0.23
-RUN pip wheel -w /wheels uwsgi==$UWSGI_VERSION
-
-COPY requirements/main.txt /requirements.txt
-RUN pip wheel -w /wheels -r /requirements.txt
+# Common WSGI middleware modules to be pip-installed
+# These are not required in every Giftless installation but are common enough
+ARG EXTRA_PACKAGES="wsgi_cors_middleware"
+# expose shared ARGs
+ARG WORKDIR
+ARG VENV
+
+# Set WORKDIR (also creates the dir)
+WORKDIR $WORKDIR
+
+# Install packages to build wheels for uWSGI and other requirements
+RUN set -eux ;\
+    export DEBIAN_FRONTEND=noninteractive ;\
+    apt-get update ;\
+    apt-get install -y --no-install-recommends build-essential libpcre3 libpcre3-dev git ;\
+    rm -rf /var/lib/apt/lists/*
+
+# Install uv to replace pip & friends
+COPY --from=uv /uv /uvx /bin/
+
+# Set a couple uv-related settings
+# Wait a bit longer for slow connections
+ENV UV_HTTP_TIMEOUT=100
+# Don't cache packages
+ENV UV_NO_CACHE=1
+
+# Create virtual env to store dependencies, "activate" it
+RUN uv venv "$VENV"
+ENV VIRTUAL_ENV="$VENV" PATH="$VENV/bin:$PATH"
+
+# Install runtime dependencies
+RUN --mount=target=/build-ctx \
+    uv pip install -r /build-ctx/requirements/main.txt
+RUN uv pip install uwsgi==$UWSGI_VERSION
+# Install extra packages into the virtual env
+RUN uv pip install ${EXTRA_PACKAGES}
+
+# Copy project contents necessary for an editable install
+COPY .git .git/
+COPY giftless giftless/
+COPY pyproject.toml .
+# Editable-install the giftless package (add a kind of a project path reference in site-packages)
+# To detect the package version dynamically, setuptools-scm needs the git binary
+RUN uv pip install -e .
 
 ### --- Build Final Image ---
-
-FROM python:3.12-slim
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get update \
-    && apt-get install -y libpcre3 libxml2 tini git \
-    && apt-get clean \
-    && apt -y autoremove
-
-RUN mkdir /app
-
-# Install dependencies
-COPY --from=builder /wheels /wheels
-RUN pip install /wheels/*.whl
-
-# Copy project code
-COPY . /app
-RUN pip install -e /app
+FROM python:3.12-slim AS final
+LABEL org.opencontainers.image.authors="Shahar Evron <shahar.evron@datopian.com>"
 
 ARG USER_NAME=giftless
+# Writable path for local LFS storage
 ARG STORAGE_DIR=/lfs-storage
-ENV GIFTLESS_TRANSFER_ADAPTERS_basic_options_storage_options_path $STORAGE_DIR
-
-RUN useradd -d /app $USER_NAME
-RUN mkdir $STORAGE_DIR
-RUN chown $USER_NAME $STORAGE_DIR
-
-# Pip-install some common WSGI middleware modules
-# These are not required in every Giftless installation but are common enough
-ARG EXTRA_PACKAGES="wsgi_cors_middleware"
-RUN pip install ${EXTRA_PACKAGES}
-
+# expose shared ARGs
+ARG WORKDIR
+ARG VENV
+
+# Set WORKDIR (also creates the dir)
+WORKDIR $WORKDIR
+
+# Create a user and set local storage write permissions
+RUN set -eux ;\
+    useradd -d "$WORKDIR" "$USER_NAME" ;\
+    mkdir "$STORAGE_DIR" ;\
+    chown "$USER_NAME" "$STORAGE_DIR"
+
+# Install runtime dependencies
+RUN set -eux ;\
+    export DEBIAN_FRONTEND=noninteractive ;\
+    apt-get update ;\
+    apt-get install -y libpcre3 libxml2 tini ;\
+    rm -rf /var/lib/apt/lists/*
+
+# Use the virtual env with dependencies from builder stage
+COPY --from=builder "$VENV" "$VENV"
+ENV VIRTUAL_ENV="$VENV" PATH="$VENV/bin:$PATH"
+# Copy project source back into the same path referenced by the editable install
+COPY --from=builder "$WORKDIR/giftless" "giftless"
+
+# Set runtime properties
 USER $USER_NAME
+ENV GIFTLESS_TRANSFER_ADAPTERS_basic_options_storage_options_path="$STORAGE_DIR"
+ENV UWSGI_MODULE="giftless.wsgi_entrypoint"
 
-WORKDIR /app
-
-ENV UWSGI_MODULE "giftless.wsgi_entrypoint"
-
-ENTRYPOINT ["tini", "uwsgi", "--"]
+ENTRYPOINT ["tini", "--", "uwsgi"]
 CMD ["-s", "127.0.0.1:5000", "-M", "-T", "--threads", "2", "-p", "2", \
      "--manage-script-name", "--callable", "app"]
 

Makefile

@@ -5,8 +5,8 @@ TESTS_DIR := tests
 
 SHELL := bash
 PYTHON := python
-PIP := pip
-PIP_COMPILE := pip-compile
+PIP := uv pip
+PIP_COMPILE := uv pip compile
 PYTEST := pytest
 DOCKER := docker
 GIT := git
@@ -28,6 +28,9 @@ VERSION := $(shell $(PYTHON) -c "from importlib.metadata import version;print(ve
 
 default: help
 
+## Install uv (fast pip replacement)
+init: $(SENTINELS)/uv
+
 ## Regenerate requirements files
 requirements: requirements/dev.txt requirements/dev.in requirements/main.txt requirements/main.in
 
@@ -90,6 +93,10 @@ $(SENTINELS)/dev-setup: init requirements/main.txt requirements/dev.txt | $(SENT
 	$(PIP) install -r requirements/dev.txt
 	@touch $@
 
+$(SENTINELS)/uv: $(SENTINELS)
+	pip install uv
+	@touch $@
+
 # Help related variables and targets
 
 GREEN  := $(shell tput -Txterm setaf 2)

README.md

@@ -5,7 +5,7 @@ Giftless - a Pluggable Git LFS Server
 [![Maintainability](https://api.codeclimate.com/v1/badges/58f05c5b5842c8bbbdbb/maintainability)](https://codeclimate.com/github/datopian/giftless/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/58f05c5b5842c8bbbdbb/test_coverage)](https://codeclimate.com/github/datopian/giftless/test_coverage)
 
-Giftless a Python implementation of a [Git LFS][1] Server. It is designed
+Giftless is a Python implementation of a [Git LFS][1] Server. It is designed
 with flexibility in mind, to allow pluggable storage backends, transfer
 methods and authentication methods.
 
@@ -40,7 +40,7 @@ Documentation
 
 License
 -------
-Copyright (C) 2020, Datopian / Viderum, Inc.
+Copyright (C) 2020-2024, Datopian / Viderum, Inc.
 
 Giftless is free / open source software and is distributed under the terms of
 the MIT license. See [LICENSE](LICENSE) for details.

docs/source/auth-providers.md

@@ -196,27 +196,44 @@ servers.
 This authenticator lets you provide a frictionless LFS backend for existing GitHub repositories. It plays nicely with `git` credential helpers and allows you to use GitHub as the single authentication & authorization provider.
 
 ### Details
-The authenticator uses [GitHub Personal Access Tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens), the same ones used for cloning a GitHub repo over HTTPS. The provided token is used in a couple GitHub API calls that identify the token's identity and [its permissions](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#get-repository-permissions-for-a-user) for the GitHub organization & repository. The token is supposed to be passed in the password part of the `Basic` HTTP auth (username is ignored). `Bearer` token HTTP auth is also supported, although no git client will likely use it.
+The authenticator uses GitHub [Personal Access Tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) and [App Installation tokens](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation), the same ones used for cloning a GitHub repo over HTTPS. The provided token is used in a couple GitHub API calls that identify the token's identity and [its permissions](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#get-repository-permissions-for-a-user) for the GitHub organization & repository.
 
-For the authenticator to work properly the token must have the `read:org` for "Classic" or `metadata:read` permission for the fine-grained kind.
-
- Note: Authentication via SSH that could be used to verify the user is [not possible with GitHub at the time of writing](https://github.com/datopian/giftless/issues/128#issuecomment-2037190728).
+Note: Authentication via SSH that could be used to verify the user is [not possible with GitHub at the time of writing](https://github.com/datopian/giftless/issues/128#issuecomment-2037190728).
 
 The GitHub repository permissions are mapped to [Giftless permissions](#permissions) in the straightforward sense that those able to write will be able to write, same with read; invalid tokens or identities with no repository access will get rejected.
 
 To minimize the traffic to GitHub for each LFS action, most of the auth data is being temporarily cached in memory, which improves performance, but naturally also ignores immediate changes for identities with changed permissions.
 
 ### GitHub Auth Flow
-Here's a description of the authentication & authorization flow. If any of these steps fails, the request gets rejected.
+Here's a description of the authentication & authorization flow. If any of these steps fails, the request gets rejected. As the supported token flavors have very different ways of authentication, they're described separately:
+
+#### Personal Access Tokens (`ghp_`, `_github_pat_` and likely other [token flavors](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats) `gho_`, `ghu_`)
+These tokens eventually represent a real user. For the authenticator to work properly, the token must have these permissions:
+- `read:org` for "Classic" or
+- `metadata:read` for the fine-grained kind.
+- The user has to be a collaborator on the target repository with an adequate role for reading or writing.
 
-1. The URI of the primary git LFS (HTTP) [`batch` request](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md) is used (as usual) to determine what GitHub organization and repository is being targeted (e.g. `https://<server>/<org>/<repo>.git/info/lfs/...`). The request's `Authentication` header is also searched for the required GitHub personal access token.
+1. The URI of the primary git LFS (HTTP) [`batch` request](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md) is used to determine what GitHub organization and repository is being targeted (e.g. `https://<server>/<org>/<repo>.git/info/lfs/...`). The request's `Authentication` header is searched for the required token in the `password` part of the `Basic` HTTP auth.
 2. The token is then used in a [`/user`](https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-the-authenticated-user) GitHub API call to get its identity data.
 3. Further on the GitHub API is asked for the [user's permissions](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#get-repository-permissions-for-a-user) to the org/repo in question.
 4. Based on the information above the user will be granted or rejected access.
 
+#### App Installation Tokens (`ghs_`)
+This token represents a special identity of an "application installation", acting on behalf of an installed GitHub App (likely part of an automation integration). This installation is bound to a user or organization (owner) and gets a set of fine-grained permissions applying to `all` or `selected` repositories of the targeted owner. For the authenticator to work properly, the GitHub App must have these permissions:
+- `metadata:read` (default)
+- `contents:read|write` (the permission to the repository content)
+- `organization_administration:read` (required to [list owner's app installations](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-app-installations-for-an-organization))
+- The installed App also has to have access to the target repository.
+
+1. The URI of the primary git LFS (HTTP) [`batch` request](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md) is used to determine what GitHub organization and repository is being targeted (e.g. `https://<server>/<org>/<repo>.git/info/lfs/...`). The request's `Authentication` header is searched for the required token in the `password` part of the `Basic` HTTP auth. **The `user` part must contain some identification of the app** (installation `id`, `app_id`, `client_id` or `app_slug` (its dashed name)).
+2. The token is then used in the [`/orgs/<org>/installations`](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-app-installations-for-an-organization) GitHub API call to get the list of app installations in the target `org`. This list is then searched for the app identification from the `user` part above. The identified entry contains info about the app permissions and whether the installation targets `all` repositories or just `selected`. At this moment the LFS permissions are inferred from the provided `content` permission. If the repository access is `all`, this is everything the logic needs.
+3. If the repository access is just for `selected` ones, the GitHub API is asked for the [`/installation/repositories`](https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-repositories-accessible-to-the-app-installation), where it must find the target repository.
+
 ### `giftless.auth.github` Configuration Options
 * `api_url` (`str` = `"https://api.github.com"`): Base URL for the GitHub API (enterprise servers have API at `"https://<custom-hostname>/api/v3/"`).
+* `api_timeout` (`float | tuple[float, float]` = `(10.0, 20.0)`): Timeout for the GitHub API calls ([details](https://requests.readthedocs.io/en/stable/user/advanced/#timeouts)).
 * `api_version` (`str | None` = `"2022-11-28"`): Target GitHub API version; set to `None` to use GitHub's latest (rather experimental).
+* `restrict_to` (`dict[str, list[str] | None] | None` = `None`): Optional (but highly recommended) dictionary of GitHub organizations/users the authentication is restricted to. Each key (organization name) in the dictionary can contain a list of further restricted repository names. When the list is empty (or null), only the organizations are considered.
 * `cache` (`dict`): Cache configuration section
   * `token_max_size` (`int` = `32`): Max number of entries in the token -> user LRU cache. This cache holds the authentication data for a token. Evicted tokens will need to be re-authenticated.
   * `auth_max_size` (`int` = `32`): Max number of [un]authorized org/repos TTL(LRU) for each user. Evicted repos will need to get re-authorized.

docs/source/installation.md

@@ -59,7 +59,7 @@ $ git clone https://github.com/datopian/giftless.git
 $ cd giftless
 $ python3 -m venv venv
 $ source venv/bin/activate
-(venv) $ pip install -r requirements.txt
+(venv) $ pip install -r requirements/main.txt
 ```
 
 You can then proceed to run Giftless with a WSGI server as