From ad57edcddaf03e8e82963ea000b45bd17b3cfe05 Mon Sep 17 00:00:00 2001 From: Danila Danko Date: Sat, 22 Jul 2023 19:08:35 +0300 Subject: [PATCH] upd workflow --- .github/workflows/ci.yaml | 385 ++++++++++++++++++--------------- app_purescript/default.nix | 15 ++ app_purescript/flake.nix | 6 +- app_python/default.nix | 15 ++ app_python/flake.nix | 5 +- flake.lock | 6 +- flake.nix | 5 +- nix-files/commands.nix | 4 +- nix-files/default.nix | 16 +- nix-files/docs.nix | 12 +- nix-files/github/ci.nix | 299 ++++++++++--------------- nix-files/tasks.nix | 12 +- nix-files/terraform/docker.nix | 4 +- nix-files/terraform/github.nix | 4 +- nix-files/terraform/yc.nix | 4 +- nix-files/write-configs.nix | 30 +-- 16 files changed, 397 insertions(+), 425 deletions(-) create mode 100644 app_purescript/default.nix create mode 100644 app_python/default.nix diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d8921bc90..441c83f64 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,26 +1,149 @@ jobs: - app-ci-app_purescript: + _1_init: + name: Initial Nix CI + permissions: + actions: write + contents: write + runs-on: ubuntu-22.04 + steps: + - name: Checkout this repo + uses: actions/checkout@v3 + - name: Install Nix + uses: nixbuild/nix-quick-install-action@v25 + with: + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + substituters = https://cache.nixos.org/ https://nix-community.cachix.org https://cache.iog.io https://deemp.cachix.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= deemp.cachix.org-1:9shDxyR2ANqEPQEEYDL/xIOnoPwxHot21L5fiZnFL18= + keep-outputs = true + - name: Restore and cache Nix store + uses: nix-community/cache-nix-action@v1 + with: + key: nix-ubuntu-22.04-job-${{ hashfiles('**/flake.nix', '**/flake.lock') }} + linux-gc-enabled: true + linux-max-store-size: 0 + restore-keys: | + nix-ubuntu-22.04-job- + - name: Remove old cache profiles + run: rm -rf /nix/var/nix/profiles/cache + - name: Configure git for github-actions + run: | + git config --global user.name github-actions + git config --global user.email github-actions@github.com + - name: Update flake locks + run: |- + nix profile install .#updateLocks + nix run .#updateLocks + - name: Format Nix files + run: |- + nix profile install .#format + nix run .#format + - name: Commit & Push + run: |- + git pull --rebase --autostash + + git add . + + git commit \ + -m "action" \ + -m "Format Nix files" \ + -m "Update flake locks" \ + || echo "commit failed!" + + git push + - name: Purge cache + uses: deemp/purge-cache@v1 + with: + accessed: false + created: true + debug: true + max-age: 172800 + _2_changed-files-app_purescript: + name: Check if app_purescript has any modified files + needs: + - _1_init + outputs: + app_purescript: ${{ steps.changed-files.outputs.any_modified }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout this repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@v37 + with: + files: app_purescript/** + sha: ${{ github.sha }} + - if: steps.changed-files.outputs.any_modified == 'true' + name: List changed files + run: echo "${{ steps.changed-files.outputs.all_changed_files }}" + _2_changed-files-app_python: + name: Check if app_python has any modified files + needs: + - _1_init + outputs: + app_python: ${{ steps.changed-files.outputs.any_modified }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout this repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@v37 + with: + files: app_python/** + sha: ${{ github.sha }} + - if: steps.changed-files.outputs.any_modified == 'true' + name: List changed files + run: echo "${{ steps.changed-files.outputs.all_changed_files }}" + _3_app-ci-app_purescript: defaults: run: working-directory: app_purescript - if: needs.changed-files-app_purescript.outputs.app_purescript == 'true' + if: needs._2_changed-files-app_purescript.outputs.app_purescript == 'true' name: CI for app_purescript needs: - - changed-files-app_purescript - runs-on: ubuntu-20.04 + - _2_changed-files-app_purescript + permissions: + actions: write + contents: write + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - name: Checkout workflows repo + - name: Checkout this repo uses: actions/checkout@v3 + - name: Install Nix + uses: nixbuild/nix-quick-install-action@v25 with: - path: .actions - repository: deemp/workflows - - name: Prepare Nix - uses: ./.actions/.github/actions/prepare-nix + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + substituters = https://cache.nixos.org/ https://nix-community.cachix.org https://cache.iog.io https://deemp.cachix.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= deemp.cachix.org-1:9shDxyR2ANqEPQEEYDL/xIOnoPwxHot21L5fiZnFL18= + keep-outputs = true + - name: Restore and cache Nix store + uses: nix-community/cache-nix-action@v1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + key: nix-${{ matrix.os }}-job-${{ hashfiles('**/flake.nix', '**/flake.lock') }} + linux-gc-enabled: true + linux-max-store-size: 0 + macos-gc-enabled: true + macos-max-store-size: 0 + restore-keys: | + nix-${{ matrix.os }}-job- + - name: Remove old cache profiles + run: rm -rf /nix/var/nix/profiles/cache + - if: matrix.os == 'ubuntu-22.04' + name: Configure git for github-actions + run: | + git config --global user.name github-actions + git config --global user.email github-actions@github.com - name: Build app - run: nix run .#build + run: |- + nix profile install .#build + nix run .#build - continue-on-error: true env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} @@ -28,30 +151,66 @@ jobs: uses: snyk/actions/node@master with: args: --all-projects - - name: Test - run: nix run .#test - app-ci-app_python: + - name: Test app + run: |- + nix profile install .#test + nix run .#test + - name: Purge cache + uses: deemp/purge-cache@v1 + with: + accessed: false + created: true + debug: true + max-age: 172800 + strategy: + matrix: + os: + - ubuntu-22.04 + - macos-12 + _3_app-ci-app_python: defaults: run: working-directory: app_python - if: needs.changed-files-app_python.outputs.app_python == 'true' + if: needs._2_changed-files-app_python.outputs.app_python == 'true' name: CI for app_python needs: - - changed-files-app_python - runs-on: ubuntu-20.04 + - _2_changed-files-app_python + permissions: + actions: write + contents: write + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - name: Checkout workflows repo + - name: Checkout this repo uses: actions/checkout@v3 + - name: Install Nix + uses: nixbuild/nix-quick-install-action@v25 with: - path: .actions - repository: deemp/workflows - - name: Prepare Nix - uses: ./.actions/.github/actions/prepare-nix + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + substituters = https://cache.nixos.org/ https://nix-community.cachix.org https://cache.iog.io https://deemp.cachix.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= deemp.cachix.org-1:9shDxyR2ANqEPQEEYDL/xIOnoPwxHot21L5fiZnFL18= + keep-outputs = true + - name: Restore and cache Nix store + uses: nix-community/cache-nix-action@v1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Linting - run: nix run .#lint + key: nix-${{ matrix.os }}-job-${{ hashfiles('**/flake.nix', '**/flake.lock') }} + linux-gc-enabled: true + linux-max-store-size: 0 + macos-gc-enabled: true + macos-max-store-size: 0 + restore-keys: | + nix-${{ matrix.os }}-job- + - name: Remove old cache profiles + run: rm -rf /nix/var/nix/profiles/cache + - if: matrix.os == 'ubuntu-22.04' + name: Configure git for github-actions + run: | + git config --global user.name github-actions + git config --global user.email github-actions@github.com + - name: Lint app + run: |- + nix profile install .#lint + nix run .#lint - continue-on-error: true env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} @@ -59,146 +218,32 @@ jobs: uses: snyk/actions/python@master with: args: --all-projects - - name: Test - run: nix run .#test - caching-macos-11: - name: Caching on macos-11 - runs-on: macos-11 - steps: - - uses: actions/checkout@v3 - - name: Checkout workflows repo - uses: actions/checkout@v3 - with: - path: .actions - repository: deemp/workflows - - name: Prepare Nix - uses: ./.actions/.github/actions/prepare-nix + - name: Test app + run: |- + nix profile install .#test + nix run .#test + - name: Purge cache + uses: deemp/purge-cache@v1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Log in to Cachix - uses: cachix/cachix-action@v12 - with: - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - name: ${{ secrets.CACHIX_CACHE }} - - name: Cache flakes - run: CACHIX_CACHE=${{ secrets.CACHIX_CACHE }} nix run .#pushToCachix - caching-macos-12: - name: Caching on macos-12 - runs-on: macos-12 - steps: - - uses: actions/checkout@v3 - - name: Checkout workflows repo - uses: actions/checkout@v3 - with: - path: .actions - repository: deemp/workflows - - name: Prepare Nix - uses: ./.actions/.github/actions/prepare-nix - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Log in to Cachix - uses: cachix/cachix-action@v12 - with: - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - name: ${{ secrets.CACHIX_CACHE }} - - name: Cache flakes - run: CACHIX_CACHE=${{ secrets.CACHIX_CACHE }} nix run .#pushToCachix - caching-ubuntu-20: - name: Caching on ubuntu-20.04 - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - name: Checkout workflows repo - uses: actions/checkout@v3 - with: - path: .actions - repository: deemp/workflows - - name: Prepare Nix - uses: ./.actions/.github/actions/prepare-nix - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Configure git - run: | - git config user.name github-actions - git config user.email github-actions@github.com - - name: Update locks - run: "git pull --rebase --autostash\nnix run .#updateLocks \ngit diff --exit-code || git commit -a -m 'action: update flake.lock-s'\ngit push\n\n" - - name: Log in to Cachix - uses: cachix/cachix-action@v12 - with: - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - name: ${{ secrets.CACHIX_CACHE }} - - name: Cache flakes - run: CACHIX_CACHE=${{ secrets.CACHIX_CACHE }} nix run .#pushToCachix - caching-ubuntu-22: - name: Caching on ubuntu-22.04 - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - name: Checkout workflows repo - uses: actions/checkout@v3 - with: - path: .actions - repository: deemp/workflows - - name: Prepare Nix - uses: ./.actions/.github/actions/prepare-nix - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Log in to Cachix - uses: cachix/cachix-action@v12 - with: - authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} - name: ${{ secrets.CACHIX_CACHE }} - - name: Cache flakes - run: CACHIX_CACHE=${{ secrets.CACHIX_CACHE }} nix run .#pushToCachix - changed-files-app_purescript: - name: Check if app_purescript has any modified files - outputs: - app_purescript: ${{ steps.changed-files.outputs.any_modified }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: changed-files - name: Get changed files - uses: tj-actions/changed-files@v32 - with: - files: app_purescript/** - sha: ${{ github.sha }} - - if: steps.changed-files.outputs.any_modified == 'true' - name: List changed files - run: | - echo "One or more files in the docs folder has changed." - echo "List all the files that have changed: ${{ steps.changed-files.outputs.all_changed_files }}" - changed-files-app_python: - name: Check if app_python has any modified files - outputs: - app_python: ${{ steps.changed-files.outputs.any_modified }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - id: changed-files - name: Get changed files - uses: tj-actions/changed-files@v32 - with: - files: app_python/** - sha: ${{ github.sha }} - - if: steps.changed-files.outputs.any_modified == 'true' - name: List changed files - run: | - echo "One or more files in the docs folder has changed." - echo "List all the files that have changed: ${{ steps.changed-files.outputs.all_changed_files }}" - push-to-docker-hub-app_purescript: - if: needs.changed-files-app_purescript.outputs.app_purescript == 'true' + accessed: false + created: true + debug: true + max-age: 172800 + strategy: + matrix: + os: + - ubuntu-22.04 + - macos-12 + _4_push-to-docker-hub-app_purescript: name: Push 'app_purescript' to Docker Hub needs: - - changed-files-app_purescript - runs-on: ubuntu-20.04 + - _3_app-ci-app_purescript + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - name: Checkout this repo + uses: actions/checkout@v3 + - name: Pull latest repo changes + run: git pull --rebase --autostash - name: Hadolint Action uses: hadolint/hadolint-action@v2.0.0 with: @@ -220,14 +265,16 @@ jobs: context: app_purescript push: true tags: ${{ secrets.DOCKER_HUB_USERNAME }}/app_purescript:latest - push-to-docker-hub-app_python: - if: needs.changed-files-app_python.outputs.app_python == 'true' + _4_push-to-docker-hub-app_python: name: Push 'app_python' to Docker Hub needs: - - changed-files-app_python - runs-on: ubuntu-20.04 + - _3_app-ci-app_python + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - name: Checkout this repo + uses: actions/checkout@v3 + - name: Pull latest repo changes + run: git pull --rebase --autostash - name: Hadolint Action uses: hadolint/hadolint-action@v2.0.0 with: @@ -249,7 +296,7 @@ jobs: context: app_python push: true tags: ${{ secrets.DOCKER_HUB_USERNAME }}/app_python:latest -name: Caching +name: CI "on": pull_request: {} push: {} diff --git a/app_purescript/default.nix b/app_purescript/default.nix new file mode 100644 index 000000000..68a8b6308 --- /dev/null +++ b/app_purescript/default.nix @@ -0,0 +1,15 @@ +(import + ( + let + lock = (builtins.fromJSON (builtins.readFile ../flake.lock)).nodes.flakes.locked; + in + ( + import + "${fetchTarball { + url = "https://github.com/deemp/flakes/archive/${lock.rev}.tar.gz"; + sha256 = lock.narHash; + }}/source-flake" + ).outputs.flake-compat.outPath + ) + { src = ./.; } +).defaultNix diff --git a/app_purescript/flake.nix b/app_purescript/flake.nix index 8ddda57db..4af1d8202 100644 --- a/app_purescript/flake.nix +++ b/app_purescript/flake.nix @@ -40,7 +40,7 @@ source ${./app.env} npx parcel serve -p $PORT --host $HOST dev/index.html ''; - description = "run app"; + description = "Run app"; }; test = { # https://github.com/mozilla/geckodriver/releases/tag/v0.31.0 @@ -57,12 +57,12 @@ kill $parcel_pid || echo "test finished" ''; runtimeInputs = tools ++ [ pkgs.poetry pkgs.geckodriver pkgs.firefox ]; - description = "test app"; + description = "Test app"; }; build = { text = build; runtimeInputs = tools; - description = "lint app"; + description = "Build app"; }; }; devShells.default = mkShell { diff --git a/app_python/default.nix b/app_python/default.nix new file mode 100644 index 000000000..68a8b6308 --- /dev/null +++ b/app_python/default.nix @@ -0,0 +1,15 @@ +(import + ( + let + lock = (builtins.fromJSON (builtins.readFile ../flake.lock)).nodes.flakes.locked; + in + ( + import + "${fetchTarball { + url = "https://github.com/deemp/flakes/archive/${lock.rev}.tar.gz"; + sha256 = lock.narHash; + }}/source-flake" + ).outputs.flake-compat.outPath + ) + { src = ./.; } +).defaultNix diff --git a/app_python/flake.nix b/app_python/flake.nix index 18ddac02f..98bb085bf 100644 --- a/app_python/flake.nix +++ b/app_python/flake.nix @@ -58,8 +58,6 @@ runtimeInputs = [ pkgs.poetry ]; }; }; - in - { devShells.default = mkShell { packages = [ pkgs.poetry ]; bash.extra = activateVenv; @@ -67,6 +65,9 @@ mkRunCommands "scripts" { inherit (packages) run-start test lint; } ++ mkCommands "tools" tools; }; + in + { + inherit packages devShells; }); in outputs; diff --git a/flake.lock b/flake.lock index 03a11eda3..ff124c5b9 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "flakes": { "locked": { - "lastModified": 1689968597, - "narHash": "sha256-QDscPCHKbe8DQlOeHgrS7+REscA0VzgBAeGBWkSxxxE=", + "lastModified": 1690037392, + "narHash": "sha256-cr4iL2lvH97QxKjOAcN5WVBq05uZKcnDE4DUs5/Lhgc=", "owner": "deemp", "repo": "flakes", - "rev": "7c1a9819ae524ca3c8b983ed1f5ed95e2499df34", + "rev": "70a7a9ee1b51707f7687d8da17d15fcb8f4cbd4b", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 71c4977c1..aed28b802 100644 --- a/flake.nix +++ b/flake.nix @@ -11,8 +11,8 @@ inherit (flakes) codium drv-tools devshell flakes-tools workflows env2json json2md; python-tools = flakes.language-tools.python; purescript-tools = flakes.language-tools.purescript; - app-python = import ./app_python; - app-purescript = import ./app_purescript; + app_python = import ./app_python; + app_purescript = import ./app_purescript; inherit flakes; }; @@ -27,6 +27,7 @@ in { inherit devShells packages; + formatter = inputs.formatter.${system}; }); in outputs; diff --git a/nix-files/commands.nix b/nix-files/commands.nix index 053bbd04e..0b337f77b 100644 --- a/nix-files/commands.nix +++ b/nix-files/commands.nix @@ -1,7 +1,7 @@ -{ pkgs, scripts, system, drv-tools }: +{ pkgs, scripts, system, inputs }: let inherit (import ./data.nix) dockerPorts langs commandNames serviceNames langPython langPurescript appName; - inherit (drv-tools.lib.${system}) mkShellApp mkBin mkShellApps; + inherit (inputs.drv-tools.lib.${system}) mkShellApp mkBin mkShellApps; # we assume that the commands will start in the corresponding directories mkCommands = lang: diff --git a/nix-files/default.nix b/nix-files/default.nix index fa39b0276..7644f45d6 100644 --- a/nix-files/default.nix +++ b/nix-files/default.nix @@ -1,7 +1,6 @@ { inputs , system , root -, }: let pkgs = inputs.nixpkgs.legacyPackages.${system}; @@ -13,20 +12,13 @@ let inherit (import ./data.nix) appPurescript appPython langPurescript langPython; inherit (builtins) map attrValues; - writeConfigs = - (import ./write-configs.nix - { - inherit (inputs) json2md env2json terrafix drv-tools workflows codium; - inherit system pkgs commands; - } - ); + writeConfigs = import ./write-configs.nix { inherit system pkgs commands inputs; }; commands = import ./commands.nix { - inherit pkgs system; - inherit (inputs) drv-tools; + inherit pkgs system inputs; scripts = { - ${langPurescript} = inputs.app-purescript.scripts.${system}; - ${langPython} = inputs.app-python.scripts.${system}; + ${langPurescript} = inputs.${appPurescript}.scripts.${system}; + ${langPython} = inputs.${appPython}.scripts.${system}; }; }; diff --git a/nix-files/docs.nix b/nix-files/docs.nix index d4c1b001f..b12abd731 100644 --- a/nix-files/docs.nix +++ b/nix-files/docs.nix @@ -1,12 +1,6 @@ -{ pkgs, env2json, system }: +{ pkgs, inputs, system }: let - inherit (import ./data.nix) - appName - langs - commandNames - taskNames - actionNames - ; + inherit (import ./data.nix) appName langs commandNames taskNames actionNames; runTask = task: "`Command palette` -> `Tasks: Run Task` -> `${task}`"; nixDevelop = command: "`${command}`"; link = title: source: "[${title}](${source})"; @@ -79,7 +73,7 @@ in ( let actionNames_ = actionNames.apps // { stop = "stop"; }; - mkJSON = env2json.functions.${system}.mkJSON; + mkJSON = inputs.env2json.lib.${system}.mkJSON; ports = builtins.foldl' (a: b: a // b) { } (builtins.map (lang: diff --git a/nix-files/github/ci.nix b/nix-files/github/ci.nix index 72dcb841c..050a011f0 100644 --- a/nix-files/github/ci.nix +++ b/nix-files/github/ci.nix @@ -1,58 +1,13 @@ -{ appPurescript, appPython, pkgs, drv-tools, workflows, system }: +{ appPurescript, appPython, pkgs, system, inputs }: let - inherit (drv-tools.lib.${system}) mergeValues; - inherit (drv-tools.lib.${system}) mkAccessors genAttrsId; - inherit (workflows.lib.${system}) expr names; - inherit (pkgs.lib.attrsets) genAttrs mapAttrsRecursive; + inherit (inputs.drv-tools.lib.${system}) mergeValues mkAccessors genAttrsId singletonIf; + inherit (inputs.workflows.lib.${system}) expr steps job job_ names os run; + inherit (pkgs.lib.attrsets) genAttrs mapAttrsRecursive recursiveUpdate; + inherit (pkgs.lib.lists) flatten imap0; - name = "Caching"; - ubuntu20 = "ubuntu-20.04"; - ubuntu22 = "ubuntu-22.04"; - ubuntu20_ = "ubuntu-20"; - ubuntu22_ = "ubuntu-22"; - macos12 = "macos-12"; - macos11 = "macos-11"; - CACHIX_CACHE_ = "CACHIX_CACHE"; - mainOS = ubuntu20; - configGitActions = '' - git config user.name github-actions - git config user.email github-actions@github.com - ''; + name = "CI"; apps = [ appPurescript appPython ]; - oss = [ ubuntu20 ubuntu22 ]; - - insertListIf = cond: list: if cond then list else [ ]; - changed-files-app = app: "changed-files-${app}"; - - actions = { - logInToCachix = { - name = "Log in to Cachix"; - uses = "cachix/cachix-action@v12"; - "with" = { - name = expr names.secrets.CACHIX_CACHE; - authToken = expr names.secrets.CACHIX_AUTH_TOKEN; - }; - }; - installNix = [ - { - name = "Checkout workflows repo"; - uses = "actions/checkout@v3"; - "with" = { - repository = "deemp/workflows"; - path = ".actions"; - }; - } - { - name = "Prepare Nix"; - uses = "./.actions/.github/actions/prepare-nix"; - "with" = { - GITHUB_TOKEN = expr names.secrets.GITHUB_TOKEN; - }; - } - ]; - checkout = { uses = "actions/checkout@v3"; }; - }; on = { schedule = [ @@ -65,7 +20,56 @@ let workflow_dispatch = { }; }; - app-ci = + mkJobApp = jobId: app: "${jobId}-${app}"; + + jobInit.name = "_1_init"; + jobInit.value = job { + runsOn = os.ubuntu-22; + name = "Initial Nix CI"; + strategy = { }; + doSaveFlakes = false; + doUpdateLocks = true; + doFormat = true; + doCommit = true; + cacheNixArgs = { + linuxGCEnabled = true; + # linuxMaxStoreSize = 0; + }; + }; + + jobChangedFiles.name = "_2_changed-files"; + jobChangedFiles.value = + let stepId = "changed-files"; in + genAttrs apps (app: + { + name = "Check if ${app} has any modified files"; + runs-on = os.ubuntu-22; + needs = [ jobInit.name ]; + outputs = { + "${app}" = expr "steps.${stepId}.outputs.any_modified"; + }; + steps = [ + (steps.checkout // { "with".fetch-depth = 0; }) + { + name = "Get changed files"; + id = stepId; + uses = "tj-actions/changed-files@v37"; + "with" = { + sha = expr names.github.sha; + files = "${app}/**"; + }; + } + { + name = "List changed files"; + "if" = "steps.${stepId}.outputs.any_modified == 'true'"; + run = ''echo "${expr "steps.${stepId}.outputs.all_changed_files" }"''; + } + ]; + } + ); + + jobCI.name = "_3_app-ci"; + jobCI.value = let matrix = { "${appPurescript}" = { @@ -83,48 +87,45 @@ let }; in builtins.mapAttrs - (app: val: - let changed-files-app_ = changed-files-app app; in + (app: appMatrix: + let jobId = mkJobApp jobChangedFiles.name app; in { - name = "CI for ${app}"; - needs = [ changed-files-app_ ]; - "if" = "needs.${changed-files-app_}.outputs.${app} == 'true'"; + needs = [ jobId ]; + "if" = "needs.${jobId}.outputs.${app} == 'true'"; defaults = { run = { working-directory = app; }; }; - runs-on = "ubuntu-20.04"; - steps = [ - { uses = "actions/checkout@v3"; } - ] - ++ - actions.installNix - ++ - ( - # - No need to lint PureScript since it's strongly statically typed - insertListIf (app == appPython) [ - { - name = "Linting"; - run = "nix run .#lint"; + } // + (job { + name = "CI for ${app}"; + strategy.matrix.os = [ os.ubuntu-22 os.macos-12 ]; + doCacheNix = true; + cacheNixArgs = { + linuxGCEnabled = true; + # linuxMaxStoreSize = 0; + macosGCEnabled = true; + # macosMaxStoreSize = 0; + }; + steps = _: [ + ( + # No need to lint PureScript since it's strongly statically typed + singletonIf (app == appPython) { + name = "Lint app"; + run = run.nixScript { name = inputs.${app}.packages.${system}.lint.pname; }; } - ] - ) - ++ - ( - # No need to build app_python as it's an interpretable lang - insertListIf (app == appPurescript) [ - { + ) + ( + # No need to build app_python as it's an interpretable lang + singletonIf (app == appPurescript) { name = "Build app"; - run = "nix run .#build"; + run = run.nixScript { name = inputs.${app}.packages.${system}.build.pname; }; } - ] - ) - ++ - [ + ) { - name = "Run Snyk to check for vulnerabilities ${val.snyk.language-title}"; - uses = "snyk/actions/${ val.snyk.language }@master"; + name = "Run Snyk to check for vulnerabilities ${appMatrix.snyk.language-title}"; + uses = "snyk/actions/${ appMatrix.snyk.language }@master"; continue-on-error = true; "with" = { args = "--all-projects"; @@ -134,115 +135,32 @@ let }; } { - name = "Test"; - run = "nix run .#test"; + name = "Test app"; + run = run.nixScript { name = inputs.${app}.packages.${system}.test.pname; }; } ]; - }) - matrix; - changed-files_ = "changed-files"; - changed-files = - genAttrs apps (app: - { - name = "Check if ${app} has any modified files"; - runs-on = ubuntu20; - outputs = { - "${app}" = expr "steps.${changed-files_}.outputs.any_modified"; - }; - steps = [ - (actions.checkout // { - "with" = { fetch-depth = 0; }; - }) - { - name = "Get changed files"; - id = changed-files_; - uses = "tj-actions/changed-files@v32"; - "with" = { - sha = expr names.github.sha; - files = "${app}/**"; - }; - } - { - name = "List changed files"; - "if" = "steps.${changed-files_}.outputs.any_modified == 'true'"; - run = '' - echo "One or more files in the docs folder has changed." - echo "List all the files that have changed: ${expr "steps.${changed-files_}.outputs.all_changed_files" }" - ''; - } - ]; - } - ); - - caching = - let - gitNixAction = { actionName, action, args ? "" }: '' - git pull --rebase --autostash - nix run ${action} ${args} - git diff --exit-code || git commit -a -m 'action: ${actionName}' - git push - ''; - matrix = { - "${ubuntu20_}" = ubuntu20; - "${ubuntu22_}" = ubuntu22; - "${macos11}" = macos11; - "${macos12}" = macos12; - }; - in - builtins.mapAttrs - (os_: os: - { - name = ''Caching on ${ os }''; - runs-on = os; - steps = [ - actions.checkout - ] - ++ - actions.installNix - ++ - (insertListIf (os == mainOS) - [ - { - name = "Configure git"; - run = configGitActions; - } - { - name = "Update locks"; - run = '' - ${gitNixAction { actionName = "update flake.lock-s"; action = ".#updateLocks"; }} - ''; - } - ] - ) ++ [ - actions.logInToCachix - { - name = "Cache flakes"; - run = "${CACHIX_CACHE_}=${expr names.secrets.CACHIX_CACHE } nix run .#pushToCachix"; - } - ]; - } - ) + })) matrix; - # a.purs.ab - push-to-docker-hub = + jobDocker.name = "_4_push-to-docker-hub"; + jobDocker.value = genAttrs apps (app: - let changed-files-app_ = changed-files-app app; in + let jobId = mkJobApp jobCI.name app; in { name = "Push '${app}' to Docker Hub"; - needs = [ changed-files-app_ ]; - "if" = "needs.${changed-files-app_}.outputs.${app} == 'true'"; - runs-on = ubuntu20; + needs = [ jobId ]; + runs-on = os.ubuntu-22; steps = [ - actions.checkout + steps.checkout + steps.gitPull { name = "Hadolint Action"; uses = "hadolint/hadolint-action@v2.0.0"; "with" = { no-fail = true; verbose = true; - dockerfile = "${ app }/Dockerfile"; + dockerfile = "${app}/Dockerfile"; }; } { @@ -274,23 +192,24 @@ let }); # app-ci.appPurescript - mkJobs = jobs_@{ ... }: mergeValues (mergeValues ( + mkJobs = jobs_@{ ... }: (mergeValues ( builtins.mapAttrs ( - name: val: { - "${name}" = mergeValues ( - builtins.mapAttrs - (name_: val_: { - "${name}-${name_}" = val_; - }) - val - ); - } + _: val: mergeValues ( + builtins.mapAttrs + (name_: val_: { + "${val.name}-${name_}" = val_; + }) + val.value + ) ) jobs_ )); - jobs = mkJobs { - inherit app-ci changed-files push-to-docker-hub caching; + jobs = mkJobs + { + inherit jobChangedFiles jobCI jobDocker; + } // { + ${jobInit.name} = jobInit.value; }; in { diff --git a/nix-files/tasks.nix b/nix-files/tasks.nix index 96068928b..136afd35a 100644 --- a/nix-files/tasks.nix +++ b/nix-files/tasks.nix @@ -1,13 +1,7 @@ -{ commands, drv-tools, system }: +{ commands, system }: let - inherit (import ./data.nix) - commandNames - langs - taskNames - actionNames - appName - ; - inherit (drv-tools.functions.${system}) mkBin; + inherit (import ./data.nix) commandNames langs taskNames actionNames appName; + inherit (inputs.drv-tools.lib.${system}) mkBin; tasksLang = lang: let taskNames_ = taskNames.apps lang; diff --git a/nix-files/terraform/docker.nix b/nix-files/terraform/docker.nix index cddb954ee..2718d8afe 100644 --- a/nix-files/terraform/docker.nix +++ b/nix-files/terraform/docker.nix @@ -1,5 +1,5 @@ -{ pkgs, system, terrafix }: -with (terrafix.hcl.${system} { inherit pkgs; }); +{ pkgs, system, inputs }: +with (inputs.terrafix.hcl.${system} { inherit pkgs; }); let inherit (import ../data.nix) appPurescript appPython; apps = [ appPurescript appPython ]; diff --git a/nix-files/terraform/github.nix b/nix-files/terraform/github.nix index 8313164ab..33ebe11fa 100644 --- a/nix-files/terraform/github.nix +++ b/nix-files/terraform/github.nix @@ -1,5 +1,5 @@ -{ pkgs, terrafix, system }: -with (terrafix.hcl.${system} { inherit pkgs; }); +{ pkgs, system }: +with (inputs.terrafix.hcl.${system} { inherit pkgs; }); let variables = mkVariables { token = b { diff --git a/nix-files/terraform/yc.nix b/nix-files/terraform/yc.nix index c9e2c6f39..341b3b124 100644 --- a/nix-files/terraform/yc.nix +++ b/nix-files/terraform/yc.nix @@ -1,5 +1,5 @@ -{ pkgs, system, terrafix }: -with (terrafix.hcl.${system} { inherit pkgs; }); +{ pkgs, system }: +with (inputs.terrafix.hcl.${system} { inherit pkgs; }); let inherit (builtins) toString; variables = mkVariables { diff --git a/nix-files/write-configs.nix b/nix-files/write-configs.nix index dbab85b46..c1047e62b 100644 --- a/nix-files/write-configs.nix +++ b/nix-files/write-configs.nix @@ -1,38 +1,32 @@ -{ json2md +{ inputs , system -, drv-tools -, workflows , pkgs , commands -, env2json -, codium -, terrafix }: let - inherit (drv-tools.lib.${system}) + inherit (inputs.drv-tools.lib.${system}) mkShellApp writeJSON framedBrackets mkBin concatStringsNewline mkBinName writeYAML concatMapStringsNewline withMan indentStrings4 ; - man = drv-tools.lib.${system}.man; - inherit (codium.lib.${system}) settingsNix; - inherit (codium.lib.${system}) writeSettingsJSON writeTasksJSON; + man = inputs.drv-tools.lib.${system}.man; + inherit (inputs.codium.lib.${system}) settingsNix writeSettingsJSON writeTasksJSON; inherit (import ./data.nix) commandNames taskNames appPurescript appPython DOCKER_PORT HOST_PORT; + inherit (inputs.json2md.lib.${system}) nix2md; inherit (builtins) map; - inherit (json2md.lib.${system}) nix2md; # all scripts assume calling from the $PROJECT_ROOT - writeDocs = nix2md "README/docs.md" (import ./docs.nix { inherit pkgs env2json system; }); + writeDocs = nix2md "README/docs.md" (import ./docs.nix { inherit pkgs inputs system; }); writeMarkdownlintConfig = writeJSON "markdownlint" ".markdownlint.jsonc" (import ./markdownlint-config.nix); writeSettings = writeSettingsJSON (import ./settings.nix { inherit settingsNix pkgs mkBinName; }); - writeTasks = writeTasksJSON (import ./tasks.nix { inherit commands drv-tools system; }); + writeTasks = writeTasksJSON (import ./tasks.nix { inherit commands inputs system; }); writeTerraform = let - inherit (terrafix.lib.${system}) writeFiles; - docker = import ./terraform/docker.nix { inherit pkgs system terrafix; }; - yc = import ./terraform/yc.nix { inherit pkgs system terrafix; }; - github = import ./terraform/github.nix { inherit pkgs system terrafix; }; + inherit (inputs.terrafix.lib.${system}) writeFiles; + docker = import ./terraform/docker.nix { inherit pkgs system inputs; }; + yc = import ./terraform/yc.nix { inherit pkgs system inputs; }; + github = import ./terraform/github.nix { inherit pkgs system inputs; }; dirDocker = "terraform/docker"; dirYC = "terraform/yandex-cloud"; dirGithub = "terraform/github"; @@ -49,7 +43,7 @@ let ]; writeWorkflows = writeYAML "workflows" ".github/workflows/ci.yaml" ( - import ./github/ci.nix { inherit appPurescript appPython pkgs drv-tools workflows system; } + import ./github/ci.nix { inherit appPurescript appPython pkgs system inputs; } ); writelib =