Merge pull request #380 from lsst-sqre/tickets/DM-44145/app-metrics

DM-45394: Config from a single YAML file (except secrets)
lsst-sqre · Nov 12, 2024 · 3e9f712 · 3e9f712
2 parents a4fb8ff + 987765e
commit 3e9f712
Show file tree

Hide file tree

Showing 45 changed files with 913 additions and 809 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -7,7 +7,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.7.2
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

diff --git a/Dockerfile b/Dockerfile
@@ -61,4 +61,4 @@ USER appuser
 EXPOSE 8080
 
 # Run the application.
-CMD ["uvicorn", "mobu.main:app", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["uvicorn", "mobu.main:create_app", "--host", "0.0.0.0", "--port", "8080"]
diff --git a/changelog.d/20241112_163347_danfuchs_app_metrics.md b/changelog.d/20241112_163347_danfuchs_app_metrics.md
@@ -0,0 +1,5 @@
+<!-- Delete the sections that don't apply -->
+
+### Backwards-incompatible changes
+
+- All app config, including autostart config (and excluding secrets, which still come from env vars) now comes from a single YAML file, provisioned by a single `ConfigMap` in Phalanx.
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -0,0 +1 @@
+../CHANGELOG.md
diff --git a/docs/development/idfdev.rst b/docs/development/idfdev.rst
@@ -17,56 +17,85 @@ You can run mobu locally while having all of the actual business run against ser
       set -euo pipefail
 
       config_dir="/tmp/mobu_test"
-      ci_config_file="github.yaml"
-      ci_config_path="$config_dir/$ci_config_file"
-      autostart_config_file="autostart.yaml"
-      autostart_config_path="$config_dir/$autostart_config_file"
+      config_file="mobu_config.yaml"
+      config_path="$config_dir/$config_file"
 
       mkdir -p "$config_dir"
 
       # Note: This whitespace must be actual <tab> chars!
-      cat <<- 'END' > "$ci_config_path"
-              users:
+      cat <<-'END' >"$config_path"
+          logLevel: debug
+          githubRefreshApp:
+            acceptedGithubOrgs:
+            - lsst-sqre
+          githubCiApp:
+            users:
               - username: bot-mobu-ci-local-1
               - username: bot-mobu-ci-local-2
-              accepted_github_orgs:
+            scopes:
+              - "exec:notebook"
+              - "exec:portal"
+              - "read:image"
+              - "read:tap"
+            acceptedGithubOrgs:
               - lsst-sqre
+          autostart:
+            - name: "my-test"
+              count: 1
+              users:
+                - username: "bot-mobu-my-test-local"
+              scopes:
+                - "exec:notebook"
+              business:
+                type: "NotebookRunner"
+                options:
+                  repo_url: "https://github.com/lsst-sqre/dfuchs-test-mobu.git"
+                  repo_ref: "dfuchs-test-pr"
+                  max_executions: 10
+                restart: true
+            - name: "my-other-test"
+              count: 1
+              users:
+                - username: "bot-mobu-my-test-local2"
+              scopes:
+                - "exec:notebook"
+              business:
+                type: "NotebookRunner"
+                options:
+                  repo_url: "https://github.com/lsst-sqre/dfuchs-test-mobu.git"
+                  repo_ref: "main"
+                  max_executions: 10
+                restart: true
+            - name: "dfuchs-test-tap"
+              count: 1
+              users:
+                - username: "bot-mobu-dfuchs-test-tap"
+              scopes: ["read:tap"]
+              business:
+                type: "TAPQuerySetRunner"
+                options:
+                  query_set: "dp0.2"
+                restart: true
+            - name: "tap"
+              count: 1
+              users:
+                - username: "bot-mobu-dfuchs-test-tap-query"
+              scopes: ["read:tap"]
+              business:
+                type: "TAPQueryRunner"
+                options:
+                  queries:
+                    - "SELECT TOP 10 * FROM  TAP_SCHEMA.tables"
+                restart: true
       END
 
-      # Note: This whitespace must be actual <tab> chars!
-      cat <<- 'END' > "$autostart_config_path"
-              - name: "my-test"
-                count: 1
-                users:
-                  - username: "bot-mobu-my-test-local"
-                scopes:
-                  - "exec:notebook"
-                business:
-                  type: "NotebookRunner"
-                  options:
-                    repo_url: "https://github.com/lsst-sqre/dfuchs-test-mobu.git"
-                    repo_ref: "main"
-                    max_executions: 10
-                  restart: true
-      END
-
+      export MOBU_CONFIG_PATH="$config_path"
       export MOBU_ENVIRONMENT_URL=https://data-dev.lsst.cloud
       export MOBU_GAFAELFAWR_TOKEN=$(op read "op://Employee/data-dev.lsst.cloud personal token/credential")
-      export MOBU_AUTOSTART_PATH="$autostart_config_path"
-      export MOBU_LOG_LEVEL=debug
-
-      # Don't set the MOBU_GITHUB_REFRESH* vars if you don't need that integration
-      export MOBU_GITHUB_REFRESH_ENABLED=true
       export MOBU_GITHUB_REFRESH_APP_WEBHOOK_SECRET=$(op read "op://RSP data-dev.lsst.cloud/mobu/github-refresh-app-webhook-secret")
-
-      # Don't set the MOBU_GITHUB_REFRESH* vars if you don't need that integration
-      export MOBU_GITHUB_CI_APP_ENABLED=true
       export MOBU_GITHUB_CI_APP_WEBHOOK_SECRET=$(op read "op://RSP data-dev.lsst.cloud/mobu/github-ci-app-webhook-secret")
       export MOBU_GITHUB_CI_APP_ID=$(op read "op://RSP data-dev.lsst.cloud/mobu/github-ci-app-id")
       export MOBU_GITHUB_CI_APP_PRIVATE_KEY=$(op read "op://RSP data-dev.lsst.cloud/mobu/github-ci-app-private-key" | base64 -d)
+      export UVICORN_PORT=8001
 
-      # Don't set MOBU_GITHUB_CONFIG_PATH if you don't need any of the GitHub integrations.
-      export MOBU_GITHUB_CONFIG_PATH="$ci_config_path"
-
-      uvicorn mobu.main:app 2>&1
-
+      uvicorn mobu.main:create_app 2>&1
diff --git a/docs/index.rst b/docs/index.rst
@@ -7,6 +7,11 @@
    development/index
    api
 
+.. toctree::
+   :hidden:
+
+   changelog
+
 ####
 Mobu
 ####

diff --git a/docs/operations/github_ci_app.rst b/docs/operations/github_ci_app.rst
@@ -40,7 +40,7 @@ In :samp:`applications/mobu/values-{env}.yaml`, add a ``config.githubCiApp`` val
 .. code:: yaml
 
    config:
-     github:
+     githubCiApp:
        acceptedGithubOrgs:
          - lsst-sqre
        users:
@@ -58,7 +58,7 @@ In :samp:`applications/mobu/values-{env}.yaml`, add a ``config.githubCiApp`` val
 
 All items are required.
 
-``accepted_github_orgs``
+``acceptedGithubOrgs``
     A list of GitHub organizations from which this instance of Mobu will accept webhook requests.
     Webhook requests from any orgs not in this list will get a ``403`` response.
 

diff --git a/docs/operations/github_refresh_app.rst b/docs/operations/github_refresh_app.rst
@@ -43,6 +43,6 @@ In :samp:`applications/mobu/values-{env}.yaml`, add a ``config.githubRefreshApp`
 
 All of these items are required.
 
-``accepted_github_orgs``
+``acceptedGithubOrgs``
     A list of GitHub organizations from which this instance of Mobu will accept webhook requests.
     Webhook requests from any orgs not in this list will get a ``403`` response.
diff --git a/pyproject.toml b/pyproject.toml
@@ -28,6 +28,8 @@ monkeyflocker = "monkeyflocker.cli:main"
 [project.urls]
 Homepage = "https://mobu.lsst.io"
 Source = "https://github.com/lsst-sqre/mobu"
+"Change log" = "https://mobu.lsst.io/changelog.html"
+"Issue tracker" = "https://github.com/lsst-sqre/mobu/issues"
 
 [build-system]
 requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=6.2"]