Merge stuff from develop branch so that we can get rid of it

flake.nix

@@ -7,7 +7,7 @@
   inputs.data-merge.url = "github:divnix/data-merge";
   inputs.capsules.url = "github:input-output-hk/devshell-capsules";
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-21.11";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-22.05";
     nixpkgs-unstable.url = "github:nixos/nixpkgs/nixpkgs-unstable";
 
     nix.url = "github:nixos/nix/2.8.1";

lib/augment-nomad-job.nix

@@ -1,5 +1,5 @@
 {nixpkgs}: let
-  pkgs = import nixpkgs {system = "x86_64-linux";};
+  pkgs = nixpkgs.legacyPackages.x86_64-linux;
   maybeAddPackage = nixpkgs.lib.mapAttrs (
     name: orig:
       orig

lib/default.nix

@@ -32,7 +32,10 @@ in rec {
 
   augmentNomadJob = import ./augment-nomad-job.nix {inherit nixpkgs;};
   mkNomadJobs = ns: envs: let
-    pkgs = import nixpkgs {system = "x86_64-linux";};
+    pkgs = import nixpkgs {
+      system = "x86_64-linux";
+      overlays = [];
+    };
   in
     builtins.mapAttrs (
       n: job: let

lib/terralib.nix

@@ -17,6 +17,8 @@
     vpc_peering_connection_id = null;
     gateway_id = null;
     vpc_endpoint_id = null;
+    carrier_gateway_id = null;
+    destination_prefix_list_id = null;
   };
 in rec {
   amis = import (nixpkgs + "/nixos/modules/virtualisation/ec2-amis.nix");

modules/envoy.nix

@@ -1,6 +1,7 @@
 { lib, config, ... }:
 let cfg = config.services.envoy;
 in {
+  disabledModules = [ "services/networking/envoy.nix" ];
   options = { services.envoy.enable = lib.mkEnableOption "Enable Envoy"; };
 
   config = { systemd.services.envoy = lib.mkIf cfg.enable { }; };

modules/promtail.nix

@@ -11,10 +11,7 @@ let
       inherit (cfg.server) grpc_listen_port;
     };
 
-    clients = [{
-      url =
-        "http://${config.cluster.nodes.monitoring.privateIP}:3100/loki/api/v1/push";
-    }];
+    inherit (cfg) clients;
 
     positions = { filename = "/var/lib/promtail/positions.yaml"; };
 
@@ -106,6 +103,17 @@ in {
     services.promtail = {
       enable = lib.mkEnableOption "Enable Promtail";
 
+      clients = lib.mkOption {
+        default = [];
+        type = with lib.types; listOf (submodule {
+            options = {
+              url = lib.mkOption {
+                type = lib.types.string;
+              };
+            };
+          });
+      };
+
       server = lib.mkOption {
         default = { };
         type = with lib.types;

modules/telegraf.nix

@@ -58,7 +58,7 @@ in {
   };
 
   ###### implementation
-  config = mkIf config.services.telegraf.enable {
+  config = mkIf cfg.enable {
     systemd.services.telegraf = {
       description = "Telegraf Agent";
       wantedBy = [ "multi-user.target" ];

modules/terraform.nix

@@ -116,41 +116,27 @@ let
       fi # manual provisioning
     '';
 
+  sshArgs = "-C -oConnectTimeout=5 -oUserKnownHostsFile=/dev/null -oNumberOfPasswordPrompts=0 -oServerAliveInterval=60 -oControlPersist=600 -oStrictHostKeyChecking=no -i ./secrets/ssh-${cfg.name}";
+  ssh = "${pkgs.openssh}/bin/ssh ${sshArgs}";
+  scp = "${pkgs.openssh}/bin/scp ${sshArgs}";
+
   localProvisionerDefaultCommand = ip:
     let
       nixConf = ''
         experimental-features = nix-command flakes
       '';
       newKernelVersion = config.boot.kernelPackages.kernel.version;
+      sshTarget = "root@${ip}";
     in ''
       set -euo pipefail
 
-      echo
-      echo Waiting for ssh to come up on port 22 ...
-      while [ -z "$(
-        ${pkgs.socat}/bin/socat \
-          -T2 stdout \
-          tcp:${ip}:22,connect-timeout=2,readbytes=1 \
-          2>/dev/null
-      )" ]
-      do
-          printf " ."
-          sleep 5
-      done
-
       sleep 1
 
       echo
       echo Waiting for host to become ready ...
-      ${pkgs.openssh}/bin/ssh -C \
-        -oUserKnownHostsFile=/dev/null \
-        -oNumberOfPasswordPrompts=0 \
-        -oServerAliveInterval=60 \
-        -oControlPersist=600 \
-        -oStrictHostKeyChecking=accept-new \
-        -i ./secrets/ssh-${cfg.name} \
-        root@${ip} \
-        "until grep true /etc/ready &>/dev/null; do sleep 1; done 2>/dev/null"
+      until ${ssh} ${sshTarget} -- grep true /etc/ready &>/dev/null; do
+        sleep 1
+      done
 
       sleep 1
 
@@ -182,20 +168,42 @@ let
 
       echo
       echo Rebooting the host to load eventually newer kernels ...
-      timeout 5 ${pkgs.openssh}/bin/ssh -C \
-        -oUserKnownHostsFile=/dev/null \
-        -oNumberOfPasswordPrompts=0 \
-        -oServerAliveInterval=60 \
-        -oControlPersist=600 \
-        -oStrictHostKeyChecking=accept-new \
-        -i ./secrets/ssh-${cfg.name} \
-        root@${ip} \
+      ${ssh} ${sshTarget} -- \
         "if [ \"$(cat /proc/sys/kernel/osrelease)\" != \"${newKernelVersion}\" ]; then \
         ${pkgs.systemd}/bin/systemctl kexec \
         || (echo Rebooting instead ... && ${pkgs.systemd}/bin/systemctl reboot) ; fi" \
       || true
     '';
 
+  localProvisionerBootstrapperCommand = ip: let
+    sshTarget = "root@${ip}";
+    sopsEncrypt =
+      "${pkgs.sops}/bin/sops --encrypt --input-type json --kms '${cfg.kms}' /dev/stdin";
+  in ''
+    if ! test -f ${relEncryptedFolder}/vault.enc.json; then
+      echo
+      echo Waiting for bootstrapping on core-1 to finish for vault /var/lib/vault/vault.enc.json ...
+      while ! ${ssh} ${sshTarget} -- test -f /var/lib/vault/vault.enc.json &>/dev/null; do
+        sleep 5
+      done
+      echo ... found /var/lib/vault/vault.enc.json
+      secret="$(${ssh} ${sshTarget} -- cat /var/lib/vault/vault.enc.json)"
+      echo "$secret" > ${relEncryptedFolder}/vault.enc.json
+      ${pkgs.git}/bin/git add ${relEncryptedFolder}/vault.enc.json
+    fi
+    if ! test -f ${relEncryptedFolder}/nomad.bootstrap.enc.json; then
+      echo
+      echo Waiting for bootstrapping on core-1 to finish for nomad /var/lib/nomad/bootstrap.token ...
+      while ! ${ssh} ${sshTarget} -- test -f /var/lib/nomad/bootstrap.token &>/dev/null; do
+        sleep 5
+      done
+      echo ... found /var/lib/nomad/bootstrap.token
+      secret="$(${ssh} ${sshTarget} -- cat /var/lib/nomad/bootstrap.token)"
+      echo "{}" | ${pkgs.jq}/bin/jq ".token = \"$secret\"" | ${sopsEncrypt} > ${relEncryptedFolder}/nomad.bootstrap.enc.json
+      ${pkgs.git}/bin/git add ${relEncryptedFolder}/nomad.bootstrap.enc.json
+    fi
+  '';
+
   cfg = config.cluster;
 
   clusterType = with lib.types;
@@ -373,7 +381,7 @@ let
             inherit cidr;
             inherit (cfg) region;
 
-            subnets = lib.pipe 3 [
+            subnets = lib.pipe (builtins.length (builtins.attrNames cfg.coreNodes)) [
               (builtins.genList lib.id)
               (map (idx: lib.nameValuePair "core-${toString (idx+1)}" {
                 cidr = net.cidr.subnet 8 idx cidr;
@@ -849,7 +857,10 @@ let
 
         localProvisioner = lib.mkOption {
           type = with lib.types; localExecType;
-          default = { protoCommand = localProvisionerDefaultCommand; };
+          default = {
+            protoCommand = localProvisionerDefaultCommand;
+            bootstrapperCommand = localProvisionerBootstrapperCommand;
+          };
         };
 
         instanceType = lib.mkOption { type = with lib.types; str; };
@@ -923,7 +934,18 @@ let
   localExecType = with lib.types;
     submodule {
       options = {
-        protoCommand = lib.mkOption { type = with lib.types; functionTo str; };
+        protoCommand = lib.mkOption {
+          type = with lib.types; functionTo str;
+          description = "Provisioner command to be applied to all nodes";
+        };
+
+        bootstrapperCommand = lib.mkOption {
+          type = with lib.types; nullOr (functionTo str);
+          default = null;
+          description = ''
+            Provisioner command to apply only to the first node, when applicable.
+          '';
+        };
 
         workingDir = lib.mkOption {
           type = with lib.types; nullOr path;

modules/terraform/aws_policies.nix

@@ -162,11 +162,7 @@ in {
 
           assumeRole = {
             effect = "Allow";
-            resources = [
-              config.cluster.coreNodes.core-1.iam.instanceProfile.tfArn
-              config.cluster.coreNodes.core-2.iam.instanceProfile.tfArn
-              config.cluster.coreNodes.core-3.iam.instanceProfile.tfArn
-            ];
+            resources = lib.mapAttrsToList (_: attrs: attrs.iam.instanceProfile.tfArn) config.cluster.coreNodes;
             actions = [ "sts:AssumeRole" ];
           };