Nix and CI Improvements (#89)

* First take on Cachix config in GitHub Actions Ideally, this should both pull *and* push from the Haskell.org Cachix cache that we set up. * Put the install Nix step back in Looks like cachix-action itself doesn't install Nix for you * Enabled manual triggering of the PR build * Added --print-build-logs flag to nix build command in CI Also switched all our Nix commands to use long flags * Added --verbose to nix build command * Moved Haskell code into its own directory and started using Niv
haskell-infra · May 27, 2021 · 415103a · 415103a
1 parent ee66679
commit 415103a
Show file tree

Hide file tree

Showing 9 changed files with 285 additions and 52 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -3,6 +3,7 @@ name: CI
 on:
   pull_request:
   push:
+  workflow_dispatch:
 
 jobs:
   build:
@@ -14,12 +15,18 @@ jobs:
       - name: Install Nix
         uses: cachix/install-nix-action@v12
 
+      - name: Cachix
+        uses: cachix/cachix-action@v10
+        with:
+          name: haskell-org
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
       - name: Build Site
         # we run the checks as a separate step
-        run: nix build -f . built -o built-site --arg doCheck false
+        run: nix build --verbose --print-build-logs --file . built -o built-site --arg doCheck false
 
       - name: Check Links
-        run: nix run --quiet -f . linkchecker -c linkchecker built-site
+        run: nix run --quiet --file . linkchecker --command linkchecker built-site
 
       - uses: actions/upload-artifact@v2
         with:

diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,6 @@ dist-newstyle
 # Nix
 result*
 built-site
+
+# Emacs
+*~
diff --git a/builder/LICENSE b/builder/LICENSE
@@ -0,0 +1,30 @@
+Copyright (c) 2019, Isaac Shapira
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of Isaac Shapira nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/builder/default.nix b/builder/default.nix
@@ -0,0 +1,37 @@
+{ sources ? import ../nix/sources.nix
+, pkgs ? import sources.nixpkgs {}
+}:
+(pkgs.haskellPackages.developPackage {
+  name = builtins.baseNameOf ./.;
+  root = pkgs.nix-gitignore.gitignoreSourcePure [
+    ../.gitignore
+    "*.markdown"
+    "*.md"
+    "*.html"
+    "templates/*"
+    "css/*"
+    "js/*"
+    "img/*"
+    ".git"
+    ".github"
+  ] ./.;
+  modifier = drv: pkgs.haskell.lib.overrideCabal drv (attrs: {
+    buildTools = with pkgs.haskellPackages; (attrs.buildTools or []) ++ [
+      cabal-install
+      ghcid
+      hakyll
+      pkgs.linkchecker
+    ];
+  });
+}).overrideAttrs (old: {
+  LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive";
+  LC_ALL = "C.UTF-8";
+  shellHook = ''
+        alias buildAndWatch="cabal configure && cabal build && cabal exec haskell-org-site -- clean && cabal exec haskell-org-site -- watch"
+        echo ""
+        echo "  Haskell.org Dev Shell"
+        echo "    \`buildAndWatch\` to serve the site, and rebuild when files change."
+        echo "    \`linkchecker\`, \`ghcid\` and \`cabal\` are provided in this environment."
+        echo ""
+      '';
+})
diff --git a/haskell-org.cabal → builder/haskell-org.cabal b/haskell-org.cabal → builder/haskell-org.cabal
diff --git a/site.hs → builder/site.hs b/site.hs → builder/site.hs
diff --git a/default.nix b/default.nix
@@ -1,60 +1,16 @@
-{ compiler ? "ghc883"
-, rev      ? "d4226e3a4b5fcf988027147164e86665d382bbfa" # from Nix 20.03 release
-, pkgs     ?
-    import (builtins.fetchTarball {
-      url    = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
-    }) {}
-, doCheck ? true
+{ doCheck ? true
+, sources ? import ./nix/sources.nix
+, pkgs ? import sources.nixpkgs {}
 }:
 
 let
-
-  gitignore = pkgs.callPackage (pkgs.fetchFromGitHub {
-      owner = "siers";
-      repo = "nix-gitignore";
-      rev = "4f2d85f2f1aa4c6bff2d9fcfd3caad443f35476e";
-      sha256 = "1vzfi3i3fpl8wqs1yq95jzdi6cpaby80n8xwnwa8h2jvcw3j7kdz";
-    }) {};
-
-  builder = (pkgs.haskell.packages.${compiler}.developPackage {
-      name = builtins.baseNameOf ./.;
-      root = gitignore.gitignoreSource [
-          "*.markdown"
-          "*.md"
-          "*.html"
-          "templates/*"
-          "css/*"
-          "js/*"
-          "img/*"
-          ".git"
-          ".github"
-        ] ./.;
-      modifier = drv: pkgs.haskell.lib.overrideCabal drv (attrs: {
-        buildTools = with pkgs.haskell.packages.${compiler}; (attrs.buildTools or []) ++ [
-          cabal-install
-          ghcid
-          hakyll
-          pkgs.linkchecker
-        ];
-      });
-    }).overrideAttrs (old: {
-      LOCALE_ARCHIVE = "${pkgs.glibcLocales}/lib/locale/locale-archive";
-      LC_ALL = "C.UTF-8";
-      shellHook = ''
-        alias buildAndWatch="cabal configure && cabal build && cabal exec haskell-org-site -- clean && cabal exec haskell-org-site -- watch"
-        echo ""
-        echo "  Haskell.org Dev Shell"
-        echo "    \`buildAndWatch\` to serve the site, and rebuild when files change."
-        echo "    \`linkchecker\`, \`ghcid\` and \`cabal\` are provided in this environment."
-        echo ""
-      '';
-    });
-
+  builder = import ./builder { inherit sources pkgs; };
   built = pkgs.stdenv.mkDerivation {
     name = "haskell.org";
     inherit doCheck;
 
-    src = gitignore.gitignoreSource [
+    src = pkgs.nix-gitignore.gitignoreSourcePure [
+      ./.gitignore
       ".git"
       "*.cabal"
       "*.hs"

diff --git a/nix/sources.json b/nix/sources.json
@@ -0,0 +1,26 @@
+{
+    "niv": {
+        "branch": "master",
+        "description": "Easy dependency management for Nix projects",
+        "homepage": "https://github.com/nmattia/niv",
+        "owner": "nmattia",
+        "repo": "niv",
+        "rev": "94080ae8286024820c570a2a24ed7c36d7ad04a9",
+        "sha256": "0wlk52zwlrq727x3z1vg9d9qq4zw62ab5jzg4068iqb6hyb0cr0w",
+        "type": "tarball",
+        "url": "https://github.com/nmattia/niv/archive/94080ae8286024820c570a2a24ed7c36d7ad04a9.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs": {
+        "branch": "nixpkgs-20.09-darwin",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "05f3800b80f159ee5ef0eccd8e31a645e6723feb",
+        "sha256": "0zx8xaa58ldvw1vw2gslnj0zldjvm2kryk3jdzv1j32m29v6302v",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/05f3800b80f159ee5ef0eccd8e31a645e6723feb.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}
diff --git a/nix/sources.nix b/nix/sources.nix
@@ -0,0 +1,174 @@
+# This file has been generated by Niv.
+
+let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      else
+        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
+
+  fetch_tarball = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      else
+        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
+
+  fetch_git = name: spec:
+    let
+      ref =
+        if spec ? ref then spec.ref else
+          if spec ? branch then "refs/heads/${spec.branch}" else
+            if spec ? tag then "refs/tags/${spec.tag}" else
+              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+    in
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
+
+  fetch_local = spec: spec.path;
+
+  fetch_builtin-tarball = name: throw
+    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=tarball -a builtin=true'';
+
+  fetch_builtin-url = name: throw
+    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=file -a builtin=true'';
+
+  #
+  # Various helpers
+  #
+
+  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
+  sanitizeName = name:
+    (
+      concatMapStrings (s: if builtins.isList s then "-" else s)
+        (
+          builtins.split "[^[:alnum:]+._?=-]+"
+            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
+        )
+    );
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources: system:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in
+      if builtins.hasAttr "nixpkgs" sources
+      then sourcesNixpkgs
+      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+        import <nixpkgs> {}
+      else
+        abort
+          ''
+            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+            add a package called "nixpkgs" to your sources.json.
+          '';
+
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file pkgs name spec
+    else if spec.type == "tarball" then fetch_tarball pkgs name spec
+    else if spec.type == "git" then fetch_git name spec
+    else if spec.type == "local" then fetch_local spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then fetch_builtin-url name
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # If the environment variable NIV_OVERRIDE_${name} is set, then use
+  # the path directly as opposed to the fetched source.
+  replace = name: drv:
+    let
+      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
+    in
+      if ersatz == "" then drv else
+        # this turns the string into an actual Nix path (for both absolute and
+        # relative paths)
+        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatMapStrings = f: list: concatStrings (map f list);
+  concatStrings = builtins.concatStringsSep "";
+
+  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
+  optionalAttrs = cond: as: if cond then as else {};
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchTarball;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      else
+        fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchurl;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      else
+        fetchurl attrs;
+
+  # Create the final "sources" from the config
+  mkSources = config:
+    mapAttrs (
+      name: spec:
+        if builtins.hasAttr "outPath" spec
+        then abort
+          "The values in sources.json should not have an 'outPath' attribute"
+        else
+          spec // { outPath = replace name (fetch config.pkgs name spec); }
+    ) config.sources;
+
+  # The "config" used by the fetchers
+  mkConfig =
+    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
+    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , system ? builtins.currentSystem
+    , pkgs ? mkPkgs sources system
+    }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+
+in
+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }