Add sha256 to source-repository-packages for hydra

This allows source-repository-package to be downloaded and replaced in
with `package: /nix/store/...` by callCabalPackageToNix so that `cabal`
and `plan-to-nix` do not need to download them.

Author: hamishmack

lib/call-cabal-project-to-nix.nix

@@ -1,7 +1,7 @@
 { dotCabal, pkgs, runCommand, nix-tools, cabal-install, ghc, hpack, symlinkJoin, cacert, index-state-hashes }:
 let defaultGhc = ghc;
     defaultCabalInstall = cabal-install;
-in { index-state ? null, index-sha256 ? null, src, ghc ? defaultGhc,
+in { index-state, index-sha256 ? null, src, ghc ? defaultGhc,
   cabal-install ? defaultCabalInstall, cabalProject ? null }:
 
 # cabal-install versions before 2.4 will generate insufficient plan information.
@@ -12,7 +12,7 @@ let
   maybeCleanedSource =
     if pkgs.lib.canCleanSource src
     then pkgs.lib.cleanSourceWith {
-      src = builtins.trace "src = ${src};" src;
+      inherit src;
       filter = path: type:
         type == "directory" ||
         pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }
@@ -21,9 +21,14 @@ let
   # Using origSrc bypasses any cleanSourceWith so that it will work when
   # access to the store is restricted.  If origSrc was already in the store
   # you can pass the project in as a string.
-  rawCabalProject = if cabalProject != null
+  rawCabalProject =
+    let origSrcDir = maybeCleanedSource.origSrc or maybeCleanedSource;
+    in if cabalProject != null
     then cabalProject
-    else builtins.readFile ((maybeCleanedSource.origSrc or maybeCleanedSource) + "/cabal.project");
+    else
+      if ((builtins.readDir origSrcDir)."cabal.project" or "") == "regular"
+        then builtins.readFile (origSrcDir + "/cabal.project")
+        else null;
 
   # Look for a index-state: field in the cabal.project file
   parseIndexState = rawCabalProject:
@@ -37,7 +42,9 @@ let
 
   index-state-found = if index-state != null
     then index-state
-    else parseIndexState rawCabalProject;
+    else if rawCabalProject != null
+      then parseIndexState rawCabalProject
+      else null;
 
   # Lookup hash for the index state we found
   index-sha256-found = if index-sha256 != null
@@ -51,13 +58,93 @@ in
     then throw "provided sha256 for index-state ${index-state-found} is null!" else true);
 
 let
+  span = pred: list:
+    let n = pkgs.lib.lists.foldr (x: acc: if pred x then acc + 1 else 0) 0 list;
+    in { fst = pkgs.lib.lists.take n list; snd = pkgs.lib.lists.drop n list; };
+
+  # Parse lines of a source-repository-package block
+  parseBlockLines = blockLines: builtins.listToAttrs (builtins.concatMap (s:
+    let pair = builtins.match " *([^:]*): *(.*)" s;
+    in pkgs.lib.optional (pair != null) (pkgs.lib.attrsets.nameValuePair
+          (builtins.head pair)
+          (builtins.elemAt pair 1))) blockLines);
+
+  fetchRepo = repo: (pkgs.fetchgit {
+    url = repo.location;
+    rev = repo.tag;
+    sha256 = repo."--sha256";
+  }) + (if repo.subdir or "" == "" then "" else "/" + repo.subdir);
+
+  # Parse a source-repository-package and fetch it if it containts
+  # a line of the form
+  #   --shar256: <<SHA256>>
+  parseBlock = block:
+    let
+      x = span (pkgs.lib.strings.hasPrefix " ") (pkgs.lib.splitString "\n" block);
+      attrs = parseBlockLines x.fst;  
+    in
+      if attrs."--sha256" or "" == ""
+        then {
+          sourceRepo = [];
+          otherText = "\nsource-repository-package\n" + block;
+        }
+        else {
+          sourceRepo = [ (fetchRepo attrs) ];
+          otherText = pkgs.lib.strings.concatStringsSep "\n" x.snd;
+        };
+
+  # Deal with source-repository-packages in a way that will work on
+  # hydra build agents (as long as a sha256 is included).
+  # Replace source-repository-package blocks that have a sha256 with
+  # packages: block containing nix sotre paths of the fetched repos.
+  replaceSoureRepos = projectFile:
+    let
+      blocks = pkgs.lib.splitString "\nsource-repository-package\n" projectFile;
+      initialText = pkgs.lib.lists.take 1 blocks;
+      repoBlocks = builtins.map parseBlock (pkgs.lib.lists.drop 1 blocks);
+      sourceRepos = pkgs.lib.lists.concatMap (x: x.sourceRepo) repoBlocks;
+      otherText = pkgs.writeText "cabal.project" (pkgs.lib.strings.concatStringsSep "\n" (
+        initialText
+        ++ (builtins.map (x: x.otherText) repoBlocks)));
+    in {
+      inherit sourceRepos;
+      makeFixedProjectFile = ''
+        cp -f ${otherText} ./cabal.project
+        chmod +w -R ./cabal.project
+        echo "packages:" >> ./cabal.project
+        mkdir -p ./.source-repository-packages
+      '' +
+        ( pkgs.lib.strings.concatStrings (
+            pkgs.lib.lists.zipListsWith (n: f: ''
+              mkdir -p ./.source-repository-packages/${builtins.toString n}
+              rsync -a --prune-empty-dirs \
+                --include '*/' --include '*.cabal' --include 'package.yaml' \
+                --exclude '*' \
+                 "${f}/" "./.source-repository-packages/${builtins.toString n}/"
+              echo "  ./.source-repository-packages/${builtins.toString n}" >> ./cabal.project
+            '')
+              (pkgs.lib.lists.range 0 ((builtins.length fixedProject.sourceRepos) - 1))
+              sourceRepos
+          )
+        );
+    };
+
+  fixedProject =
+    if rawCabalProject == null
+      then {
+        sourceRepos = [];
+        makeFixedProjectFile = "";
+      }
+      else replaceSoureRepos rawCabalProject;
+
   plan = runCommand "plan-to-nix-pkgs" {
     nativeBuildInputs = [ nix-tools ghc hpack cabal-install pkgs.rsync pkgs.git ];
-  } ''
+  } (''
     tmp=$(mktemp -d)
     cd $tmp
     cp -r ${maybeCleanedSource}/* .
     chmod +w -R .
+    ${fixedProject.makeFixedProjectFile}
     # warning: this may not generate the proper cabal file.
     # hpack allows globbing, and turns that into module lists
     # without the source available (we cleaneSourceWith'd it),
@@ -96,7 +183,7 @@ let
 
     # move pkgs.nix to default.nix ensure we can just nix `import` the result.
     mv $out/pkgs.nix $out/default.nix
-  '';
+  '');
 in
   # TODO: We really want this (symlinks) instead of copying the source over each and
   #       every time.  However this will not work with sandboxed builds.  They won't
@@ -114,13 +201,23 @@ in
   #   # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
   #   buildInputs = [ plan src ];
   # }
-  runCommand "plan-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } ''
+  runCommand "plan-to-nix-pkgs-with-src" { nativeBuildInputs = [ pkgs.rsync ]; } (''
     mkdir $out
     # todo: should we clean `src` to drop any .git, .nix, ... other irelevant files?
     rsync -a "${src}/" "$out/"
     rsync -a ${plan}/ $out/
+  '' +
+    ( pkgs.lib.strings.concatStrings (
+        pkgs.lib.lists.zipListsWith (n: f: ''
+          mkdir -p $out/.source-repository-packages/${builtins.toString n}
+          rsync -a "${f}/" "$out/.source-repository-packages/${builtins.toString n}/"
+        '')
+          (pkgs.lib.lists.range 0 ((builtins.length fixedProject.sourceRepos) - 1))
+          fixedProject.sourceRepos
+      )
+    ) + ''
     # Rsync will have made $out read only and that can cause problems when
     # nix sandboxing is enabled (since it can prevent nix from moving the directory
     # out of the chroot sandbox).
     chmod +w $out
-  ''
+  '')

test/cabal-source-repo/Main.hs

@@ -0,0 +1,4 @@
+module Main where
+
+main :: IO ()
+main = putStrLn "Hello, Haskell!"

test/cabal-source-repo/cabal.project

@@ -0,0 +1,9 @@
+packages:
+  ./
+
+source-repository-package
+  type: git
+  location: https://github.com/input-output-hk/haskell.nix.git
+  tag: bc01ebc05a8105035c9449943046b46c8364b932
+  subdir: test/cabal-simple
+  --sha256: 003lm3pm024vhbfmii7xcdd9v2rczpflxf7gdl2pyxia7p014i8z

test/cabal-source-repo/default.nix

@@ -0,0 +1,33 @@
+{ stdenv, mkCabalProjectPkgSet, callCabalProjectToNix }:
+
+with stdenv.lib;
+
+let
+  pkgSet = mkCabalProjectPkgSet {
+    plan-pkgs = import (callCabalProjectToNix {
+      index-state = "2019-04-30T00:00:00Z";
+      # reuse the cabal-simple test project
+      src = ./.;
+    });
+  };
+  packages = pkgSet.config.hsPkgs;
+in
+  stdenv.mkDerivation {
+    name = "call-cabal-project-to-nix-test";
+
+    buildCommand = ''
+      exe="${packages.use-cabal-simple.components.exes.use-cabal-simple}/bin/use-cabal-simple"
+
+      printf "checking whether executable runs... " >& 2
+      $exe
+
+      touch $out
+    '';
+
+    meta.platforms = platforms.all;
+
+    passthru = {
+      # Attributes used for debugging with nix repl
+      inherit pkgSet packages;
+    };
+  }

test/cabal-source-repo/use-cabal-simple.cabal

@@ -0,0 +1,26 @@
+cabal-version:       >=1.10
+-- Initial package description 'cabal-simple.cabal' generated by 'cabal
+-- init'.  For further documentation, see
+-- http://haskell.org/cabal/users-guide/
+
+name:                use-cabal-simple
+version:             0.1.0.0
+-- synopsis:
+-- description:
+-- bug-reports:
+license:             PublicDomain
+author:              Rodney Lorrimar
+maintainer:          rodney.lorrimar@iohk.io
+-- category:
+build-type:          Simple
+
+executable use-cabal-simple
+  main-is:             Main.hs
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       base
+                     , cabal-simple
+                     , extra
+                     , optparse-applicative
+  -- hs-source-dirs:
+  default-language:    Haskell2010

test/default.nix

@@ -20,6 +20,7 @@ in pkgs.recurseIntoAttrs {
   shell-for = haskell.callPackage ./shell-for {};
   callStackToNix = haskell.callPackage ./call-stack-to-nix {};
   callCabalProjectToNix = haskell.callPackage ./call-cabal-project-to-nix {};
+  cabal-source-repo = haskell.callPackage ./cabal-source-repo {};
 
   # Run unit tests with: nix-instantiate --eval --strict -A unit.tests
   # An empty list means success.