Remote builder configurations

flake.nix

@@ -2,7 +2,8 @@
   description = "Flake containing Bitte clusters";
 
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/43cdc5b364511eabdcad9fde639777ffd9e5bab1"; # nixos-21.05
+    nixpkgs.url =
+      "github:nixos/nixpkgs/43cdc5b364511eabdcad9fde639777ffd9e5bab1"; # nixos-21.05
     nixpkgs-core.follows = "nixpkgs";
     nixpkgs-client.follows = "nixpkgs";
 
@@ -36,11 +37,13 @@
     hydra.inputs.nixpkgs.follows = "nixpkgs";
   };
 
-  outputs = { self, hydra, nixpkgs, utils, cli, deploy, ragenix, ... }@inputs:
+  outputs =
+    { self, hydra, nixpkgs, utils, cli, deploy, ragenix, nix, ... }@inputs:
     let
 
       overlays = [
-        (_: prev: { inherit (cli.packages.${prev.system}) bitte; })
+        nix.overlay
+        (_: prev: { inherit (cli.packages."${prev.system}") bitte; })
         hydra.overlay
         deploy.overlay
         localPkgsOverlay
@@ -70,10 +73,9 @@
       hydraJobs = let
         constituents = {
           inherit (legacyPackages)
-            bitte cfssl ci-env consul cue glusterfs
-            grafana-loki haproxy haproxy-auth-request haproxy-cors nixFlakes
-            nomad nomad-autoscaler oauth2-proxy sops terraform-with-plugins
-            vault-backend vault-bin;
+            bitte cfssl ci-env consul cue glusterfs grafana-loki haproxy
+            haproxy-auth-request haproxy-cors nixFlakes nomad nomad-autoscaler
+            oauth2-proxy sops terraform-with-plugins vault-backend vault-bin;
         };
       in {
         inherit constituents;

modules/terraform.nix

@@ -56,7 +56,7 @@ let
         package = pkgs.nixFlakes;
         extraOptions = '''
           show-trace = true
-          experimental-features = nix-command flakes
+          experimental-features = nix-command flakes ca-references
         ''';
         binaryCaches = [
           "https://hydra.iohk.io"
@@ -146,7 +146,7 @@ let
       export PATH="${
         lib.makeBinPath [
           pkgs.openssh
-          pkgs.nixUnstable
+          pkgs.nix
           pkgs.git
           pkgs.mercurial
           pkgs.lsof
@@ -218,6 +218,11 @@ let
           default = { };
         };
 
+        builder = lib.mkOption {
+          type = types.str;
+          default = "monitoring";
+        };
+
         route53 = lib.mkOption {
           type = with lib.types; bool;
           default = true;

modules/vault-agent.nix

@@ -51,7 +51,7 @@ in {
         };
     };
 
-    role = lib.mkOption { type = with lib.types; enum [ "client" "core" "routing" ]; };
+    role = lib.mkOption { type = with lib.types; enum [ "client" "core" "routing" "hydra" ]; };
 
     vaultAddress = lib.mkOption {
       type = with lib.types; str;

modules/vulnix.nix

@@ -136,7 +136,7 @@ in {
         VAULT_CACERT = letsencryptCertMaterial.certChainFile;
       };
 
-      path = with pkgs; [ cfg.package vault-bin curl jq nixFlakes gitMinimal ];
+      path = with pkgs; [ cfg.package vault-bin curl jq nix gitMinimal ];
 
       script = let
         mkWhitelists = map (lib.flip lib.pipe [

overlay.nix

@@ -9,9 +9,8 @@ let
     '';
 in final: prev:
 rec {
-  inherit (inputs.nix.packages.x86_64-linux) nix;
-  nixFlakes = final.nix;
-  nixUnstable = final.nix;
+  nixFlakes = nixUnstable;
+  nixUnstable = builtins.throw "use pkgs.nix directly";
 
   nomad = inputs.nomad.defaultPackage."${final.system}";
 
@@ -185,4 +184,3 @@ rec {
   });
 
 })
-

profiles/ami-base-config.nix

@@ -5,7 +5,6 @@
   */
 
   imports = [ ./slim.nix ];
-  nix.package = pkgs.nixUnstable;
   nix.binaryCaches = [ "https://hydra.iohk.io" ];
   nix.binaryCachePublicKeys =
     [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" ];

profiles/auxiliaries/builder.nix

@@ -2,114 +2,138 @@
 let
   deployType = config.currentCoreNode.deployType or config.currentAwsAutoScalingGroup.deployType;
 
+  cfg = config.profiles.auxiliaries.builder;
+
   isSops = deployType == "aws";
   isInstance = config.currentCoreNode != null;
   isAsg = !isInstance;
-  isMonitoring = nodeName == "monitoring";
+  isAsgRemoteBuilder = nodeName == cfg.asgRemoteBuilder.nodeName;
 in {
-  secrets.generate.nix-key-file = lib.mkIf isSops ''
-    export PATH="${lib.makeBinPath (with pkgs; [ nixFlakes sops coreutils ])}"
-    esk=encrypted/nix-secret-key-file
-    ssk=secrets/nix-secret-key-file
-    if [ ! -s "$esk" ]; then
-      if [ -s "$ssk" ]; then
+  options.profiles.auxiliaries.builder = with lib; {
+    enable = mkEnableOption "builder profile" // {
+      default = nodeName == cfg.asgRemoteBuilder.nodeName || isAsg;
+    };
+
+    asgRemoteBuilder = {
+      nodeName = mkOption {
+        type = types.str;
+        description = "node name of the remote build machine for ASG clients";
+        default = config.cluster.builder;
+      };
+
+      buildMachine = mkOption {
+        type = types.attrs;
+        description = "extra `nix.buildMachines.*` options";
+        default = {};
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    secrets.generate.nix-key-file = lib.mkIf isSops ''
+      export PATH="${lib.makeBinPath (with pkgs; [ nix sops coreutils ])}"
+      esk=encrypted/nix-secret-key-file
+      ssk=secrets/nix-secret-key-file
+      if [ ! -s "$esk" ]; then
+        if [ -s "$ssk" ]; then
+          sops --encrypt --input-type binary --kms '${config.cluster.kms}' "$ssk" \
+            > "$esk.new"
+        else
+          nix key generate-secret ${config.cluster.name}-0 \
+            | sops --encrypt --input-type binary --kms '${config.cluster.kms}' /dev/stdin \
+            > "$esk.new"
+        fi
+        mv "$esk.new" "$esk"
+      fi
+      if [ ! -s "$ssk" ]; then
+        sops --decrypt --input-type binary "$ssk" > "$ssk.new"
+        mv "$ssk.new" "$ssk"
+      fi
+      epk=encrypted/nix-public-key-file
+      spk=secrets/nix-public-key-file
+      for pub in "$epk" "$spk"; do
+        if [ ! -s "$pub" ]; then
+          nix key convert-secret-to-public < "$ssk" > "$pub.new"
+          mv "$pub.new" "$pub"
+        fi
+      done
+    '';
+
+    secrets.generate.builder-ssh-key = lib.mkIf isSops ''
+      export PATH="${lib.makeBinPath (with pkgs; [ openssh sops coreutils ])}"
+      epk=encrypted/nix-builder-key.pub
+      spk=secrets/nix-builder-key.pub
+      esk=encrypted/nix-builder-key
+      ssk=secrets/nix-builder-key
+      if [ ! -s "$esk" ]; then
+        ssh-keygen -t ed25519 -f "$ssk" -P "" -C "builder@${cfg.asgRemoteBuilder.nodeName}"
         sops --encrypt --input-type binary --kms '${config.cluster.kms}' "$ssk" \
           > "$esk.new"
-      else
-        nix key generate-secret ${config.cluster.name}-0 \
-          | sops --encrypt --input-type binary --kms '${config.cluster.kms}' /dev/stdin \
-          > "$esk.new"
+        mv "$esk.new" "$esk"
       fi
-      mv "$esk.new" "$esk"
-    fi
-    if [ ! -s "$ssk" ]; then
-      sops --decrypt --input-type binary "$ssk" > "$ssk.new"
-      mv "$ssk.new" "$ssk"
-    fi
-    epk=encrypted/nix-public-key-file
-    spk=secrets/nix-public-key-file
-    for pub in "$epk" "$spk"; do
-      if [ ! -s "$pub" ]; then
-        nix key convert-secret-to-public < "$ssk" > "$pub.new"
-        mv "$pub.new" "$pub"
+      if [ ! -s "$epk" ]; then
+        cp "$spk" "$epk"
       fi
-    done
-  '';
-
-  secrets.generate.builder-ssh-key = lib.mkIf isSops ''
-    export PATH="${lib.makeBinPath (with pkgs; [ openssh sops coreutils ])}"
-    epk=encrypted/nix-builder-key.pub
-    spk=secrets/nix-builder-key.pub
-    esk=encrypted/nix-builder-key
-    ssk=secrets/nix-builder-key
-    if [ ! -s "$esk" ]; then
-      ssh-keygen -t ed25519 -f "$ssk" -P "" -C "builder@monitoring"
-      sops --encrypt --input-type binary --kms '${config.cluster.kms}' "$ssk" \
-        > "$esk.new"
-      mv "$esk.new" "$esk"
-    fi
-    if [ ! -s "$epk" ]; then
-      cp "$spk" "$epk"
-    fi
-  '';
-
-  secrets.install.builder-private-ssh-key = lib.mkIf (isAsg && isSops) {
-    source = (toString config.secrets.encryptedRoot) + "/nix-builder-key";
-    target = /etc/nix/builder-key;
-    inputType = "binary";
-    outputType = "binary";
-    script = ''
-      export PATH="${lib.makeBinPath (with pkgs; [ coreutils openssh ])}"
-      chmod 0600 /etc/nix/builder-key
-      ssh \
-        -o NumberOfPasswordPrompts=0 \
-        -o StrictHostKeyChecking=accept-new \
-        -i /etc/nix/builder-key \
-        builder@${config.cluster.coreNodes.monitoring.privateIP} echo 'trust established'
     '';
-  };
 
-  age.secrets = lib.mkIf (isAsg && !isSops) {
-    docker-passwords = {
-      file = config.age.encryptedRoot + "/ssh/builder.age";
-      path = "/etc/nix/builder-key";
-      owner = "root";
-      group = "root";
-      mode = "0600";
+    secrets.install.builder-private-ssh-key = lib.mkIf (isAsg && isSops) {
+      source = (toString config.secrets.encryptedRoot) + "/nix-builder-key";
+      target = /etc/nix/builder-key;
+      inputType = "binary";
+      outputType = "binary";
       script = ''
-        ${pkgs.openssh}/bin/ssh \
+        export PATH="${lib.makeBinPath (with pkgs; [ coreutils openssh ])}"
+        chmod 0600 /etc/nix/builder-key
+        ssh \
           -o NumberOfPasswordPrompts=0 \
           -o StrictHostKeyChecking=accept-new \
           -i /etc/nix/builder-key \
-          builder@${config.cluster.coreNodes.monitoring.privateIP} echo 'trust established'
-        mv "$src" "$out"
+          builder@${cfg.asgRemoteBuilder.nodeName} echo 'trust established'
       '';
     };
-  };
 
-  nix = {
-    distributedBuilds = isAsg;
-    maxJobs = lib.mkIf isAsg 0;
-    extraOptions = ''
-      builders-use-substitutes = true
-    '';
-    trustedUsers = lib.mkIf isMonitoring [ "root" "builder" ];
-    buildMachines = lib.optionals isAsg [{
-      hostName = config.cluster.coreNodes.monitoring.privateIP;
-      maxJobs = 5;
-      speedFactor = 1;
-      sshKey = "/etc/nix/builder-key";
-      sshUser = "builder";
-      system = "x86_64-linux";
-    }];
-  };
+    age.secrets = lib.mkIf (isAsg && !isSops) {
+      docker-passwords = {
+        file = config.age.encryptedRoot + "/ssh/builder.age";
+        path = "/etc/nix/builder-key";
+        owner = "root";
+        group = "root";
+        mode = "0600";
+        script = ''
+          ${pkgs.openssh}/bin/ssh \
+            -o NumberOfPasswordPrompts=0 \
+            -o StrictHostKeyChecking=accept-new \
+            -i /etc/nix/builder-key \
+            builder@${cfg.asgRemoteBuilder.nodeName} echo 'trust established'
+          mv "$src" "$out"
+        '';
+      };
+    };
+
+    nix = {
+      distributedBuilds = isAsg;
+      maxJobs = lib.mkIf isAsg 0;
+      extraOptions = ''
+        builders-use-substitutes = true
+      '';
+      trustedUsers = lib.mkIf isAsgRemoteBuilder [ "root" "builder" ];
+      buildMachines = lib.optionals isAsg [({
+        hostName = cfg.asgRemoteBuilder.nodeName;
+        maxJobs = 5;
+        speedFactor = 1;
+        sshKey = "/etc/nix/builder-key";
+        sshUser = "builder";
+        system = "x86_64-linux";
+      } // cfg.asgRemoteBuilder.buildMachine)];
+    };
 
-  users.extraUsers = lib.mkIf isMonitoring {
-    builder = {
-      isSystemUser = true;
-      openssh.authorizedKeys.keyFiles =
-        [ ((toString config.secrets.encryptedRoot) + "/nix-builder-key.pub") ];
-      shell = pkgs.bashInteractive;
+    users.extraUsers = lib.mkIf isAsgRemoteBuilder {
+      builder = {
+        isSystemUser = true;
+        openssh.authorizedKeys.keyFiles =
+          [ ((toString config.secrets.encryptedRoot) + "/nix-builder-key.pub") ];
+        shell = pkgs.bashInteractive;
+      };
     };
   };
 }

profiles/auxiliaries/nix.nix

@@ -1,6 +1,5 @@
 { pkgs, config, self, ... }: {
   nix = {
-    package = pkgs.nixFlakes;
     gc.automatic = true;
     gc.options = "--max-freed $((10 * 1024 * 1024))";
     optimise.automatic = true;

profiles/auxiliaries/secrets.nix

@@ -135,7 +135,7 @@ in {
   '';
 
   secrets.generate.cache = lib.mkIf (isInstance && isSops) ''
-    export PATH="${lib.makeBinPath (with pkgs; [ coreutils nixFlakes jq ])}"
+    export PATH="${lib.makeBinPath (with pkgs; [ coreutils nix jq ])}"
 
     mkdir -p secrets encrypted
 

profiles/bootstrap/default.nix

@@ -255,6 +255,12 @@ in {
             policies=default,routing \
             period=24h
 
+          vault write auth/aws/role/${config.cluster.name}-hydra \
+            auth_type=iam \
+            bound_iam_principal_arn="$arn:role/${config.cluster.name}-core" \
+            policies=default,hydra \
+            period=24h
+
           ${lib.concatStringsSep "\n" (lib.forEach config.cluster.adminNames (name: ''
             vault write "auth/aws/role/${name}" \
               auth_type=iam \

profiles/client.nix

@@ -11,7 +11,6 @@ in {
 
     ./auxiliaries/docker.nix
     ./auxiliaries/reaper.nix
-    ./auxiliaries/builder.nix
   ];
 
   services.s3-upload-flake.enable = deployType == "aws";

profiles/common.nix

@@ -12,6 +12,7 @@ in {
     ./auxiliaries/ssh.nix
     ./auxiliaries/promtail.nix
     ./auxiliaries/telegraf.nix
+    ./auxiliaries/builder.nix
   ];
 
   # avoid CVE-2021-4034 (PwnKit)

profiles/hydra.nix

@@ -0,0 +1 @@
+_: { imports = [ ./common.nix ./consul/client.nix ./vault/hydra.nix ]; }

profiles/monitoring.nix

@@ -10,7 +10,6 @@ in {
     ./consul/client.nix
     ./vault/monitoring.nix
 
-    ./auxiliaries/builder.nix
     ./auxiliaries/docker-registry.nix
     ./auxiliaries/loki.nix
     ./auxiliaries/oauth.nix