Initial nix development environment. Use Nix for OSX CI instead of ho…

…mebrew. (#9727)

* Initial nix development environment

Extract latest from release compiler binary and libgc.a only. That is enough to use bin/crystal with the new std-lib.

Allow bin/ci to run nix-shell when CI_NIX_SHELL is set

Use nix instead of brew for CI Darwin in GitHub

* Update shell.nix

Co-authored-by: Dorian Marié <dorian@dorianmarie.fr>

* Build GC for nix development environment.

Overrides the existing boehmgc package to apply patch needed for mt
support.

* Build gc with libatomics + mt patch

* Allow custom system to use 32 bits easily

Co-authored-by: Dorian Marié <dorian@dorianmarie.fr>
Co-authored-by: Kim Burgess <kim@place.technology>

.github/workflows/macos.yml

@@ -3,10 +3,8 @@ name: macOS CI
 on: [push, pull_request]
 
 env:
-  TRAVIS_OS_NAME: osx
-  LLVM_CONFIG: /usr/local/opt/llvm/bin/llvm-config
-  PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig
   SPEC_SPLIT_DOTS: 160
+  CI_NIX_SHELL: true
 
 jobs:
   test_macos:
@@ -15,6 +13,12 @@ jobs:
       - name: Download Crystal source
         uses: actions/checkout@v2
 
+      - uses: cachix/install-nix-action@v10
+      - uses: cachix/cachix-action@v6
+        with:
+          name: crystal-ci
+          signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+
       - name: Prepare System
         run: bin/ci prepare_system
 

bin/ci

@@ -46,12 +46,16 @@ on_os() {
   os="$1"
   shift
 
-  verify_environment
+  if [ -z "$CI_NIX_SHELL" ]; then
+    verify_environment
 
-  if [ "$TRAVIS_OS_NAME" = "$os" ]; then
-    echo "${@}"
-    eval "${@}"
-    return $?
+    if [ "$TRAVIS_OS_NAME" = "$os" ]; then
+      echo "${@}"
+      eval "${@}"
+      return $?
+    else
+      return 0
+    fi
   else
     return 0
   fi
@@ -65,6 +69,16 @@ on_osx() {
   fail_on_error on_os "osx" "${@}"
 }
 
+on_nix_shell() {
+  if [ -n "$CI_NIX_SHELL" ]; then
+    echo "${@}"
+    eval "${@}"
+    return $?
+  else
+    return 0
+  fi
+}
+
 on_github() {
   if [ "$GITHUB_ACTIONS" = "true" ]; then
     eval "${@}"
@@ -182,6 +196,8 @@ with_build_env() {
     CRYSTAL_CACHE_DIR="/tmp/crystal" \
     /bin/sh -c "'$command'"
 
+  on_nix_shell nix-shell --pure $CI_NIX_SHELL_ARGS --run "'TZ=$TZ $command'"
+
   on_github echo "::endgroup::"
 }
 

shell.nix

@@ -0,0 +1,151 @@
+# This nix-shell script can be used to get a complete development environment
+# for the Crystal compiler.
+#
+# You can choose which llvm version use and, on Linux, choose to use musl.
+#
+# $ nix-shell --pure
+# $ nix-shell --pure --arg llvm 10
+# $ nix-shell --pure --arg llvm 10 --arg musl true
+# $ nix-shell --pure --arg llvm 9
+# $ nix-shell --pure --arg llvm 9 --argstr system i686-linux
+# ...
+# $ nix-shell --pure --arg llvm 6
+#
+# If needed, you can use https://app.cachix.org/cache/crystal-ci to avoid building
+# packages that are not available in Nix directly. This is mostly useful for musl.
+#
+# $ nix-env -iA cachix -f https://cachix.org/api/v1/install
+# $ cachix use crystal-ci
+# $ nix-shell --pure --arg musl true
+#
+
+{llvm ? 10, musl ? false, system ? builtins.currentSystem}:
+
+let
+  nixpkgs = import (builtins.fetchTarball {
+    name = "nixpkgs-20.03";
+    url = "https://github.com/NixOS/nixpkgs/archive/2d580cd2793a7b5f4b8b6b88fb2ccec700ee1ae6.tar.gz";
+    sha256 = "1nbanzrir1y0yi2mv70h60sars9scwmm0hsxnify2ldpczir9n37";
+  }) {
+    inherit system;
+  };
+
+  pkgs = if musl then nixpkgs.pkgsMusl else nixpkgs;
+
+  genericBinary = { url, sha256 }:
+    pkgs.stdenv.mkDerivation rec {
+      name = "crystal-binary";
+      src = builtins.fetchTarball { inherit url sha256; };
+
+      # Extract only the compiler binary
+      buildCommand = ''
+        mkdir -p $out/bin
+
+        # Darwin packages use embedded/bin/crystal
+        [ ! -f "${src}/embedded/bin/crystal" ] || cp ${src}/embedded/bin/crystal $out/bin/
+
+        # Linux packages use lib/crystal/bin/crystal
+        [ ! -f "${src}/lib/crystal/bin/crystal" ] || cp ${src}/lib/crystal/bin/crystal $out/bin/
+      '';
+    };
+
+  # Hashes obtained using `nix-prefetch-url --unpack <url>`
+  latestCrystalBinary = genericBinary ({
+    x86_64-darwin = {
+      url = "https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-darwin-x86_64.tar.gz";
+      sha256 = "sha256:0gpn42xh372hw2bqfgxc9wibpbam8gn7gx3p1b8p9adydjg0zxfm";
+    };
+
+    x86_64-linux = {
+      url = "https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-linux-x86_64.tar.gz";
+      sha256 = "sha256:077pby4ylf0z831gg0hbiwxcq3g0yl0cdlybirgg8rv24a2sa7zh";
+    };
+
+    i686-linux = {
+      url = "https://github.com/crystal-lang/crystal/releases/download/0.35.1/crystal-0.35.1-1-linux-i686.tar.gz";
+      sha256 = "sha256:0nfgxjndfslyacicjy4303pvvqfg74v5fnpr4b10ss9rqakmlbgd";
+    };
+  }.${pkgs.stdenv.system});
+
+  pkgconfig = pkgs.pkgconfig;
+
+  llvm_suite = ({
+    llvm_10 = {
+      llvm = pkgs.llvm_10;
+      extra = [ pkgs.lld_10 pkgs.lldb_10 ];
+    };
+    llvm_9 = {
+      llvm = pkgs.llvm_9;
+      extra = [ ]; # lldb it fails to compile on Darwin
+    };
+    llvm_8 = {
+      llvm = pkgs.llvm_8;
+      extra = [ ]; # lldb it fails to compile on Darwin
+    };
+    llvm_7 = {
+      llvm = pkgs.llvm;
+      extra = [ pkgs.lldb ];
+    };
+    llvm_6 = {
+      llvm = pkgs.llvm_6;
+      extra = [ ]; # lldb it fails to compile on Darwin
+    };
+  }."llvm_${toString llvm}");
+
+  libatomic_ops = builtins.fetchurl {
+    url = "https://github.com/ivmai/libatomic_ops/releases/download/v7.6.10/libatomic_ops-7.6.10.tar.gz";
+    sha256 = "1bwry043f62pc4mgdd37zx3fif19qyrs8f5bw7qxlmkzh5hdyzjq";
+  };
+
+  boehmgc = pkgs.stdenv.mkDerivation rec {
+    pname = "boehm-gc";
+    version = "8.0.4";
+
+    src = builtins.fetchTarball {
+      url = "https://github.com/ivmai/bdwgc/releases/download/v${version}/gc-${version}.tar.gz";
+      sha256 = "16ic5dwfw51r5lcl88vx3qrkg3g2iynblazkri3sl9brnqiyzjk7";
+    };
+
+    patches = [
+      (pkgs.fetchpatch {
+        url = "https://github.com/ivmai/bdwgc/commit/5668de71107022a316ee967162bc16c10754b9ce.patch";
+        sha256 = "02f0rlxl4fsqk1xiq0pabkhwydnmyiqdik2llygkc6ixhxbii8xw";
+      })
+    ];
+
+    postUnpack = ''
+      mkdir $sourceRoot/libatomic_ops
+      tar -xzf ${libatomic_ops} -C $sourceRoot/libatomic_ops --strip-components 1
+    '';
+
+    configureFlags = [
+      "--disable-debug"
+      "--disable-dependency-tracking"
+      "--disable-shared"
+      "--enable-large-config"
+    ];
+
+    enableParallelBuilding = true;
+  };
+
+  stdLibDeps = with pkgs; [
+      boehmgc gmp libevent libiconv libxml2 libyaml openssl pcre zlib
+    ] ++ stdenv.lib.optionals stdenv.isDarwin [ libiconv ];
+
+  tools = [ pkgs.hostname llvm_suite.extra ];
+in
+
+pkgs.stdenv.mkDerivation rec {
+  name = "crystal-dev";
+
+  buildInputs = tools ++ stdLibDeps ++ [
+    latestCrystalBinary
+    pkgconfig
+    llvm_suite.llvm
+  ];
+
+  LLVM_CONFIG = "${llvm_suite.llvm}/bin/llvm-config";
+
+  # ld: warning: object file (.../src/ext/libcrystal.a(sigfault.o)) was built for newer OSX version (10.14) than being linked (10.12)
+  MACOSX_DEPLOYMENT_TARGET = "10.11";
+}