Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix just argument splitting on pass-through recipes with complex args #4961

Merged
merged 7 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .vale/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ IGNORE_FILES_PATTERN := ".pnpm|changelogs|projects/proposals|node_modules|.?venv
# of the current working directory. If a custom separator is specified, newlines are
# replaced with this separator before outputting the file list.
_files separator="\n":
#! /usr/bin/env sh
#! /usr/bin/env bash
files=$(
find "$PWD/.." -type f \( -name "*.md" -o -name "*.mdx" \) \
| grep -vE "{{ IGNORE_FILES_PATTERN }}"
Expand Down
22 changes: 14 additions & 8 deletions api/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,14 @@ install *args:
######

# Bring up services specific to the API profile
[positional-arguments]
up *flags:
env COMPOSE_PROFILES="api" just ../up {{ flags }}
env COMPOSE_PROFILES="api" just ../up "$@"

# Bring up services specific to the API profile, in addition to the API server
[positional-arguments]
up-extra *flags:
env COMPOSE_PROFILES="api,api_extra" just ../up {{ flags }}
env COMPOSE_PROFILES="api,api_extra" just ../up "$@"

# Wait for all profile services to be up
wait-up: up
Expand Down Expand Up @@ -103,12 +105,14 @@ pgcli db_user_pass="deploy" db_name="openledger": up
#########################

# Run Django administrative commands locally
dj-local +args="":
pdm run python manage.py {{ args }}
[positional-arguments]
dj-local *args:
pdm run python manage.py "$@"

# Run Django administrative commands inside the Docker container
dj +args="": wait-up
env DC_USER="{{ env_var_or_default("DC_USER", "opener") }}" just ../exec web python manage.py {{ args }}
[positional-arguments]
dj *args: wait-up
env DC_USER="{{ env_var_or_default("DC_USER", "opener") }}" just ../exec web python manage.py "$@"

# Get IPython shell inside the Docker container
ipython:
Expand Down Expand Up @@ -152,12 +156,14 @@ generate-docs doc="media-props" fail_on_diff="true":
#########

# Run API tests inside the Docker container
[positional-arguments]
test *args: wait-up
env DC_USER="opener" just ../exec web pytest {{ args }}
env DC_USER="opener" just ../exec web pytest "$@"

# Run API tests locally
[positional-arguments]
test-local *args:
pdm run pytest {{ args }}
pdm run pytest "$@"

# Run smoke test for the API docs
doc-test: wait-up
Expand Down
10 changes: 6 additions & 4 deletions automations/js/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ NO_COLOR := "\\033[0m"
just --list --unsorted

# Run a script
[positional-arguments]
run script *args:
pnpm --filter automations exec node src/{{ script }} {{ args }}
pnpm --filter automations exec node src/{{ script }} "${@:2}"

#################
# Weekly report #
Expand All @@ -26,14 +27,15 @@ report:
##########

render-release-drafter:
#! /usr/bin/env sh
#! /usr/bin/env bash
render_release_drafter() {
slug=$1
name=$2
confjson=$(printf '{"app": "%s"}' "$name")
just run render-jinja.js \
templates/release-drafter.yml.jinja \
.github/release-drafter-$slug.yml \
"'{ \"app\": \"$name\" }'"
.github/release-drafter-"$slug".yml \
"$confjson"
}
render_release_drafter api API
render_release_drafter ingestion_server "Ingestion Server"
Expand Down
6 changes: 4 additions & 2 deletions automations/python/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ NO_COLOR := "\\033[0m"


# Install dependencies
[positional-arguments]
install *args:
pdm install {{ args }}
pdm install "$@"

# Run a script
[positional-arguments]
run script *args:
pdm run {{ script }} {{ args }}
pdm run "$@"
9 changes: 6 additions & 3 deletions catalog/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,14 @@ install *args: check-py-version
######

# Bring up services specific to the catalog profile
[positional-arguments]
up *flags:
env COMPOSE_PROFILES="catalog" just ../up {{ flags }}
env COMPOSE_PROFILES="catalog" just ../up "$@"

# Bring up services specific to the catalog profile, except Airflow
[positional-arguments]
up-deps *flags:
env COMPOSE_PROFILES="catalog_dependencies" just ../up {{ flags }}
env COMPOSE_PROFILES="catalog_dependencies" just ../up "$@"

# Load sample data into upstream DB
init: up
Expand Down Expand Up @@ -111,8 +113,9 @@ test-session:
just _mount-test bash

# Run pytest in a test container under `SERVICE`
[positional-arguments]
test *args:
just _mount-test "bash -c \'pytest {{ args }}\'"
just _mount-test "bash -c \'pytest ""$@""\'"

#############
# Utilities #
Expand Down
3 changes: 2 additions & 1 deletion docker/cache/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ NO_COLOR := "\\033[0m"
#######

# Open the Redis CLI as an interactive REPL
[positional-arguments]
cli *args:
just ../../exec cache redis-cli {{ args }}
just ../../exec cache redis-cli "$@"

# Delete all data cached in Redis
flushall:
Expand Down
3 changes: 2 additions & 1 deletion documentation/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,9 @@ live port="50230": clean
pdm run sphinx-autobuild -b html . _serve/ --port {{ port }}

# Compile the documentation into a static site
[positional-arguments]
build *args: clean
pdm run sphinx-build {{ args }} -b html . _build/
pdm run sphinx-build "$@" -b html . _build/

# Serve the compiled docs using a simple HTTP server
serve port="50231":
Expand Down
2 changes: 1 addition & 1 deletion frontend/bin/playwright.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#! /usr/bin/env sh
#! /usr/bin/env bash
set -e

version() {
Expand Down
6 changes: 4 additions & 2 deletions frontend/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ run-img tag="openverse-frontend:local":
######

# Bring up services specific to the frontend profile
[positional-arguments]
up *flags:
env COMPOSE_PROFILES="frontend" just ../up {{ flags }}
env COMPOSE_PROFILES="frontend" just ../up "$@"

# Wait for all profile services to be up
wait-up: up
Expand All @@ -67,8 +68,9 @@ init: wait-up
cd .. && ./setup_plausible.sh

# Run a package.json script via pnpm
[positional-arguments]
run *args:
pnpm run {{ args }}
pnpm run "$@"

# Generate the specified kind of documentation
generate-docs doc="media-props" fail_on_diff="true":
Expand Down
10 changes: 7 additions & 3 deletions indexer_worker/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ install *args="--dev":
######

# Bring up services specific to the indexer worker profile
[positional-arguments]
up *flags:
env COMPOSE_PROFILES="catalog_indexer_worker" just ../up {{ flags }}
env COMPOSE_PROFILES="catalog_indexer_worker" just ../up "$@"

wait-up: up
just wait
Expand Down Expand Up @@ -77,9 +78,12 @@ curl-post data host="localhost:50281":
# Tests #
#########

# Run indexer worker tests in the container
[positional-arguments]
test *args: wait-up
just ../exec catalog_indexer_worker pytest {{ args }}
just ../exec catalog_indexer_worker pytest "$@"

# Run indexer-worker tests locally
[positional-arguments]
test-local *args:
pdm run pytest {{ args }}
pdm run pytest "$@"
54 changes: 31 additions & 23 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@ set dotenv-load := false
# @ - Quiet recipes (https://github.com/casey/just#quiet-recipes)
# _ - Private recipes (https://github.com/casey/just#private-recipes)

IS_PROD := env_var_or_default("PROD", env_var_or_default("IS_PROD", ""))
# `PROD_ENV` can be "ingestion_server" or "catalog"
PROD_ENV := env_var_or_default("PROD_ENV", "")
Comment on lines -7 to -9
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These variables were used long ago when we used to deploy the ingestion server and catalog using just. We don't do that any more, and they just add noise/complexity to this file now, so are good to remove.

IS_CI := env_var_or_default("CI", "")
DC_USER := env_var_or_default("DC_USER", "opener")

Expand Down Expand Up @@ -137,8 +134,9 @@ precommit:
fi

# Run pre-commit to lint and reformat files
[positional-arguments]
lint hook="" *files="": precommit
python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files" } }} {{ files }}
python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files {{ files }}" } }}

# Run codeowners validator locally. Only enable experimental hooks if there are no uncommitted changes.
lint-codeowners checks="stable":
Expand Down Expand Up @@ -171,14 +169,7 @@ lint-codeowners checks="stable":
# Docker #
##########

DOCKER_FILE := "-f " + (
if IS_PROD == "true" {
if PROD_ENV == "ingestion_server" { "ingestion_server/docker-compose.yml" }
else if PROD_ENV == "catalog" { "catalog/docker-compose.yml" }
else { "docker-compose.yml" }
}
else { "docker-compose.yml" }
)
DOCKER_FILE := "-f docker-compose.yml"
EXEC_DEFAULTS := if IS_CI == "" { "" } else { "-T" }

export CATALOG_PY_VERSION := `just catalog/py-version`
Expand Down Expand Up @@ -207,13 +198,15 @@ versions:
EOF

# Run `docker compose` configured with the correct files and environment
[positional-arguments]
dc *args:
@{{ if IS_CI != "" { "just env" } else { "true" } }}
env COMPOSE_PROFILES="{{ env_var_or_default("COMPOSE_PROFILES", "api,ingestion_server,frontend,catalog") }}" docker compose {{ DOCKER_FILE }} {{ args }}
env COMPOSE_PROFILES="{{ env_var_or_default("COMPOSE_PROFILES", "api,ingestion_server,frontend,catalog") }}" docker compose {{ DOCKER_FILE }} "$@"

# Build all (or specified) services
[positional-arguments]
build *args:
just dc build {{ args }}
just dc build "$@"

# List all services and their URLs and ports
@ps:
Expand All @@ -223,11 +216,12 @@ build *args:

# Also see `up` recipe in sub-justfiles
# Bring all Docker services up, in all profiles
[positional-arguments]
up *flags: env && ps
#!/usr/bin/env bash
set -eo pipefail
while true; do
if just dc up {{ if IS_CI != "" { "--quiet-pull" } else { "" } }} -d {{ flags }} ; then
if just dc up {{ if IS_CI != "" { "--quiet-pull" } else { "" } }} -d "$@" ; then
break
fi
((c++)) && ((c==3)) && break
Expand All @@ -249,8 +243,9 @@ init:
just frontend/init

# Take all Docker services down, in all profiles
[positional-arguments]
down *flags:
just dc down {{ flags }}
just dc down "$@"

# Take all services down then call the specified app's up recipe. ex.: `just dup catalog` is useful for restarting the catalog with new environment variables
dup app:
Expand All @@ -277,12 +272,14 @@ attach service:
docker attach $(just dc ps | awk '{print $1}' | grep {{ service }})

# Execute statement in service containers using Docker Compose
[positional-arguments]
exec +args:
just dc exec -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} {{ args }}
just dc exec -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} "$@"

# Execute statement in a new service container using Docker Compose
[positional-arguments]
run +args:
just dc run --rm -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} "{{ args }}"
just dc run --rm -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} "$@"

# Execute pgcli against one of the database instances
_pgcli container db_user_pass db_name db_host db_port="5432":
Expand Down Expand Up @@ -352,20 +349,31 @@ f:
just frontend/run dev

# alias for `pnpm --filter {package} run {script}`
p package script +args="":
pnpm --filter {{ package }} run {{ script }} {{ args }}
[positional-arguments]
p package script *args:
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A handful of scripts used +args="", which works, but is technically redundant. *args expresses the same thing (optional variadic arguments to gather), and perhaps much more intentionally, especially to relay that they are optional, rather than the + prefix which indicates at least one is required (a requirement nullified by the default blank that is passed).

pnpm --filter {{ package }} run {{ script }} "${@:3}"

# Run eslint with --fix and default file selection enabled; used to enable easy file overriding whilst retaining the defaults when running --all-files
eslint *files="frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js prettier.config.js tsconfig.base.json":
[positional-arguments]
eslint *args:
#! /usr/bin/env bash
just p '@openverse/eslint-plugin' build
if [[ "$@" ]]; then
files=("$@")
else
# default files
files=(frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js prettier.config.js tsconfig.base.json)
fi

pnpm exec eslint \
--ext .js,.ts,.vue,.json,.json5 \
--ignore-path .gitignore \
--ignore-path .eslintignore \
--max-warnings=0 \
--fix \
{{ files }}
"${files[@]}"

# Alias for `just packages/js/k6/run` or `just p k6 run`
[positional-arguments]
@k6 *args:
just packages/js/k6/run {{ args }}
just packages/js/k6/run "$@"
5 changes: 3 additions & 2 deletions packages/js/k6/justfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Build and run K6 script by namespace and scenario
run namespace scenario +extra_args="":
[positional-arguments]
run namespace scenario *extra_args:
pnpm run build --input src/{{ namespace }}/{{ scenario }}.test.ts
k6 run {{ extra_args }} ./dist/{{ namespace }}/{{ scenario }}.test.js
k6 run "${@:3}" ./dist/{{ namespace }}/{{ scenario }}.test.js
3 changes: 2 additions & 1 deletion packages/python/openverse-attribution/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,6 @@ NO_COLOR := "\\033[0m"
###########

# Install dependencies
[positional-arguments]
install *args:
pdm install {{ args }}
pdm install "$@"
Loading