Readme

Notice

This is one of my release framework based on nix which contains following:

• deploy-env
• deploy-config
• deploy-packer(that is the repo)

What is this

This repos provides some helper nix functions and derivations to help simplefy the deployment process for projects which is built based on nix. It serves as a CD tool based on nix for my own use and I hope it would help other.

Features

The packer has following features:

reproducable

It is based on nix and inherits its reproducability to build and pack the application and its all dependencies into one tarball.

incremental

It packs the build artifactes incrementally. The first time you may get a fully pack tarbal and next time it only pack the changed artifactes so can reduce the generated tarball size dramatically.

self-extractable tarball

It uses makeself to pack all artifactes as one self-extractable tarball and simplyfy the deployment process by just copying the generated tarball to the target machine and run it

sand-box deployment

Since the application and its all env and config all packed during build-time, only some data or intermedia files generated during run-time will be out of the /nix/store on the target, the deployment can be cleanup and rollbacked very easy.

How to use it

This repo provides some helper functions or derivations to a nix project to help simplefy the deployment process. You can follow these steps to incorporate it into your project:

1. create a release.nix in your project top level directory
2. import the deploy-env and deploy-config respectively
3. create a systemd service or a shell wrapper for your application and may use the env and config as the dependencies
4. import the deploy-packer
5. use the mk-release-packer function within the deploy-packer to generate a ready-to-run packer for the service or wrapper
6. run the generated packer and ship the generated tarball to the target

Following is a release.nix file for reference:

{ nativePkgs ? import ./default.nix { }, # the native package set
pkgs ? import ./cross-build.nix { }
, # the package set for corss build, we're especially interested in the fully static binary
site, # the site for release, the binary would deploy to it finally
phase, # the phase for release, must be "local", "test" and "production"
}:
let
  nPkgs = nativePkgs.pkgs;
  sPkgs = pkgs.x86-musl64; # for the fully static build
  lib = nPkgs.lib; # lib functions from the native package set
  pkgName = "my-runner";
  innerTarballName = lib.concatStringsSep "." [
    (lib.concatStringsSep "-" [ pkgName site phase ])
    "tar"
    "gz"
  ];

  # define some utility function for release packing ( code adapted from setup-systemd-units.nix )
  deploy-packer = import (builtins.fetchGit { url = "https://github.com/hughjfchen/deploy-packer"; }) {
    inherit lib;
    pkgs = nPkgs;
  };

  # the deployment env
  my-runner-env = (import
    (builtins.fetchGit { url = "https://github.com/hughjfchen/deploy-env"; }) {
      pkgs = nPkgs;
      modules = [
        ./site/${site}/phase/${phase}/db.nix
        ./site/${site}/phase/${phase}/db-gw.nix
        ./site/${site}/phase/${phase}/api-gw.nix
        ./site/${site}/phase/${phase}/messaging.nix
        ./site/${site}/phase/${phase}/runner.nix
      ];
    }).env;

  # app and dependent config
  my-runner-config = (import (builtins.fetchGit {
    url = "https://github.com/hughjfchen/deploy-config";
  }) {
    pkgs = nPkgs;
    modules = [
      ./site/${site}/phase/${phase}/db.nix
      ./site/${site}/phase/${phase}/db-gw.nix
      ./site/${site}/phase/${phase}/api-gw.nix
      ./site/${site}/phase/${phase}/messaging.nix
      ./site/${site}/phase/${phase}/runner.nix
    ];
    env = my-runner-env;
  }).config;

  my-runner-config-kv = nPkgs.writeTextFile {
    name = lib.concatStringsSep "-" [ pkgName "config" ];
    # generate the key = value format config, refer to the lib.generators for other formats
    text = (lib.generators.toKeyValue { }) my-runner-config.runner;
  };
  my-runner-bin-sh-paths = [
    # list the runtime dependencies, especially those cannot be determined by nix automatically
    nPkgs.wget
    nPkgs.curl
    nPkgs.xvfb-run
    nPkgs.jdk11
    nPkgs.eclipse-mat
    sPkgs.java-analyzer-runner.java-analyzer-runner-exe
  ];
  my-runner-bin-sh = nPkgs.writeShellApplication {
    name = lib.concatStringsSep "-" [ pkgName "bin" "sh" ];
    runtimeInputs = my-runner-bin-sh-paths;
    # wrap the executable, suppose it accept a --config commandl ine option to load the config
    text = ''
      ${sPkgs.java-analyzer-runner.java-analyzer-runner-exe.exeName} --config.file="${my-runner-config-kv}" "$@"
    '';
  };
  # following define the service
  my-runner-service = { lib, pkgs, config, ... }: {
    options = lib.attrsets.setAttrByPath [ "services" pkgName ] {
      enable = lib.mkOption {
        default = true;
        type = lib.types.bool;
        description = "enable to generate a config to start the service";
      };
      # add extra options here, if any
    };
    config = lib.mkIf
      (lib.attrsets.getAttrFromPath [ pkgName "enable" ] config.services)
      (lib.attrsets.setAttrByPath [ "systemd" "services" pkgName ] {
        wantedBy = [ "multi-user.target" ];
        after = [ "network.target" ];
        description = "${pkgName} service";
        serviceConfig = {
          Type = "forking";
          User = "${my-runner-env.runner.processUser}";
          ExecStart =
            "${my-runner-bin-sh}/bin/${my-runner-bin-sh.name} --command=Start";
          Restart = "on-failure";
        };
      });
  };

  serviceNameKey = lib.concatStringsSep "." [ pkgName "service" ];
  serviceNameUnit =
    lib.attrsets.setAttrByPath [ serviceNameKey ] mk-my-runner-service-unit;

  mk-my-runner-service-unit = nPkgs.writeText serviceNameKey
    (lib.attrsets.getAttrFromPath [
      "config"
      "systemd"
      "units"
      serviceNameKey
      "text"
    ] (nPkgs.nixos
      ({ lib, pkgs, config, ... }: { imports = [ my-runner-service ]; })));

in rec {
  inherit nativePkgs pkgs;
  mk-my-runner-service-systemd-setup-or-bin-sh =
    if my-runner-env.runner.isSystemdService then
      (nPkgs.setupSystemdUnits {
        namespace = pkgName;
        units = serviceNameUnit;
      })
    else
      my-runner-bin-sh;

  mk-my-runner-service-systemd-unsetup-or-bin-sh =
    if my-runner-env.runner.isSystemdService then
      (deploy-packer.unsetup-systemd-service {
        namespace = pkgName;
        units = serviceNameUnit;
      })
    else
      { };
  # following derivation just to make sure the setup and unsetup will
  # be packed into the distribute tarball.
  setup-and-unsetup-or-bin-sh = nPkgs.symlinkJoin {
    name = "my-runner-setup-and-unsetup";
    paths = [
      mk-my-runner-service-systemd-setup-or-bin-sh
      mk-my-runner-service-systemd-unsetup-or-bin-sh
    ];
  };

  mk-my-runner-reference =
    nPkgs.writeReferencesToFile setup-and-unsetup-or-bin-sh;

  mk-my-runner-deploy-sh = deploy-packer.mk-deploy-sh {
    env = my-runner-env.runner;
    payloadPath = setup-and-unsetup-or-bin-sh;
    inherit innerTarballName;
    execName = "${my-runner-bin-sh.name}";
    startCmd = "--command=Start";
    stopCmd = "--command=Stop";
  };
  mk-my-runner-cleanup-sh = deploy-packer.mk-cleanup-sh {
    env = my-runner-env.runner;
    payloadPath = setup-and-unsetup-or-bin-sh;
    inherit innerTarballName;
    execName = "${my-runner-bin-sh.name}";
  };
  mk-my-release-packer = deploy-packer.mk-release-packer {
    referencePath = mk-my-runner-reference;
    component = pkgName;
    inherit site phase innerTarballName;
    deployScript = mk-my-runner-deploy-sh;
    cleanupScript = mk-my-runner-cleanup-sh;
  };

}

You can even write some scripts to simplyfy the release process further, following is an reference:

#!/usr/bin/env bash

if ! type dirname > /dev/null 2>&1; then
    echo "Not even a linux or macOS, Windoze? We don't support it. Abort."
    exit 1
fi

. "$(dirname "$0")"/common.sh

init_with_root_or_sudo "$0"

SCRIPT_ABS_PATH=$(turn_to_absolute_path "$0")

begin_banner "Top level" "project deploy - generic"

if [ $# != 2 ]; then
    echo "usage: $(basename "$0") deployTargetSite releasePhase"
    exit 125
fi
[ -d "$SCRIPT_ABS_PATH/../env/site/$1/phase/$2" ] || (echo "Directory $SCRIPT_ABS_PATH/../env/site/$1/phase/$2 not exists" && exit 126)
[ -d "$SCRIPT_ABS_PATH/../config/site/$1/phase/$2" ] || (echo "Directory $SCRIPT_ABS_PATH/../config/site/$1/phase/$2 not exists" && exit 126)

set +u
[ -e "$HOME"/.nix-profile/etc/profile.d/nix.sh ] && . "$HOME"/.nix-profile/etc/profile.d/nix.sh
set -u

# build the boundle for the specific release target
nix-build ./release.nix --arg site \""$1"\" --arg phase \""$2"\" --attr "mk-my-release-packer" --out-link "mk-my-release-packer"

# pack the build artifact up with the dependencies
"$SCRIPT_ABS_PATH/mk-my-release-packer/bin/mk-release-packer-for-$1-$2"

done_banner "Top level" "project deploy - generic"

Make makeself fully quite

The script within the generated tarbal by makeself is not fully quiet by default, if you want to run the generated tarbal in a fully quiet mode, you can overrite the makeself derivation in the default nixpkgs as following:

self: prev:

# override the makeself package to make sure quiet mode is the default
prev.makeself.overrideAttrs (oldAttrs: {
  fixupPhase = [ oldAttrs.fixupPhase ] ++ [''
    sed -e "s|quiet=\"n\"|quiet=\"y\"|; s|accept=\"n\"|accept=\"y\"|; s|noprogress=\$NOPROGRESS|noprogress=\"y\"|" -i $out/share/makeself-2.4.2/makeself-header.sh
  ''];
})

Todos

Following enhancement may be implemented in the future release for this repos:

• integrated it into the quick-cook-haskell-project DONE