Skip to content

Reduce the number of derivations #1605

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 51 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
b4a36e9
Store default setup on ghc derivation
hamishmack Aug 5, 2022
9b06216
Less derivations first cut
hamishmack Aug 7, 2022
ad6a27a
Fixes and optimize nix-tools usage
hamishmack Aug 9, 2022
1069180
Fix cabal.config
hamishmack Aug 11, 2022
d7004ee
Fix for cabal package db issue in nix shell
hamishmack Aug 15, 2022
08bb11b
Fixes for shellFor and hoogle in shell
hamishmack Aug 15, 2022
c94ed63
Fixes for shellFor and hoogle in shell
hamishmack Aug 15, 2022
417a7da
Fix hpack support
hamishmack Aug 16, 2022
afb52d9
Fix (p.shellFor {}).ghc
hamishmack Aug 16, 2022
1aa5e2b
More fixes for shellFor
hamishmack Aug 16, 2022
16e23cb
More fixes for shellFor
hamishmack Aug 16, 2022
11fe87c
Use ghc env for exactDeps
hamishmack Aug 16, 2022
12ecc24
Keep config files and wrapped ghc for cabal-doctest
hamishmack Aug 16, 2022
51737c7
Fix name clash
hamishmack Aug 16, 2022
82cda52
Remove special treatment of ghc dependency in custom setup
hamishmack Aug 17, 2022
af62a32
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Aug 18, 2022
14f7a4b
Fix ghcjs and use ln -s instead of cp
hamishmack Aug 20, 2022
362b982
ifdLevel 0
hamishmack Aug 20, 2022
07c4426
ifdLevel 1
hamishmack Aug 20, 2022
c744c1e
ifdLevel 2
hamishmack Aug 20, 2022
0c6f255
ifdLevel 3
hamishmack Aug 20, 2022
557bebc
Fix for ghcjs default setup
hamishmack Aug 21, 2022
acb0eb0
Only propagate inputs to library components
hamishmack Aug 22, 2022
d17dcbf
ifdLevel 0
hamishmack Aug 20, 2022
872cb27
Shorten nix-tools attribute path
hamishmack Aug 22, 2022
db3ca22
Shorten nix-tools attribute path
hamishmack Aug 22, 2022
ba5bc4e
Fix closure size issue
hamishmack Aug 22, 2022
46846bf
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Aug 25, 2022
42d08a6
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Aug 29, 2022
e894bc2
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Aug 31, 2022
9499917
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Aug 31, 2022
5f32a61
ifdLevel 1
hamishmack Aug 31, 2022
eb6e1c2
Use cabal 3.8
hamishmack Sep 2, 2022
0dedd34
ifdLevel 0
hamishmack Sep 2, 2022
eccd186
Use tool function for cabal
hamishmack Sep 2, 2022
bbcc46f
ifdLevel 1
hamishmack Sep 3, 2022
1d7f07d
ifdLevel 2
hamishmack Sep 3, 2022
a17863c
ifdLevel 3
hamishmack Sep 3, 2022
2611a66
Explicit pkg-config
hamishmack Sep 4, 2022
0acd595
Merge branch 'hkm/use-cabal-3.8' into hkm/combined-derivations
hamishmack Sep 4, 2022
8a0d0d1
Fix merge issue
hamishmack Sep 4, 2022
eec5710
ifdLevel 2
hamishmack Sep 4, 2022
204296f
More fixes and work arounds
hamishmack Sep 5, 2022
a39ad04
Fix for cross compile
hamishmack Sep 5, 2022
0e3417a
Fix test.sh
hamishmack Sep 6, 2022
9621d93
Add description for `pkgconfigSelector`
hamishmack Sep 6, 2022
09fec30
Merge branch 'hkm/use-cabal-3.8' into hkm/combined-derivations
hamishmack Sep 6, 2022
b5f42a5
Fix merge issue
hamishmack Sep 6, 2022
8021d30
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Oct 3, 2022
489111a
Merge remote-tracking branch 'origin/master' into hkm/combined-deriva…
hamishmack Oct 5, 2022
db5eff6
Check that stack builds
hamishmack Oct 5, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixes for shellFor and hoogle in shell
  • Loading branch information
hamishmack committed Aug 15, 2022
commit 08bb11b47604eb66e04e09c0a3469d9c6792ddaa
1 change: 1 addition & 0 deletions builder/comp-builder.nix
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,7 @@ let
if id=$(${target-pkg-and-db} field "z-${package.identifier.name}-z-*" id --simple-output); then
name=$(${target-pkg-and-db} field "z-${package.identifier.name}-z-*" name --simple-output)
echo "--dependency=''${name#z-${package.identifier.name}-z-}=$id" >> $out/exactDep/configure-flags
echo "package-id $id" >> $out/envDep
''
# Allow `package-name:sublib-name` to work in `build-depends`
# by adding the same `--dependency` again, but with the package
Expand Down
14 changes: 8 additions & 6 deletions builder/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ let


hoogleLocal = let
nixpkgsHoogle = import (pkgs.path + /pkgs/development/haskell-modules/hoogle.nix);
# Use hoogle.nix from at least nixpkgs 22.05
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should just throw an error if it's before then? Otherwise it's probably surprising to people that this might pull in a whole extra nixpkgs?

nixpkgs = if lib.versionAtLeast lib.trivial.release "22.05"
then pkgs.path
else pkgs.haskell-nix.sources.nixpkgs-2205;
nixpkgsHoogle = import (nixpkgs + /pkgs/development/haskell-modules/hoogle.nix);
in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.tool "ghc8107" "hoogle" {
inherit evalPackages;
version = "5.0.18.3";
Expand All @@ -59,11 +63,9 @@ let
else ghc;
inherit packages hoogle;
};
in if lib.versionAtLeast lib.trivial.release "22.05"
then haskellLib.weakCallPackage pkgs nixpkgsHoogle {
inherit haskellPackages;
} (p: p.packages)
else haskellLib.weakCallPackage pkgs nixpkgsHoogle haskellPackages;
in haskellLib.weakCallPackage pkgs nixpkgsHoogle {
inherit haskellPackages;
} (p: p.packages);

# Same as haskellPackages.shellFor in nixpkgs.
shellFor = haskellLib.weakCallPackage pkgs ./shell-for.nix {
Expand Down
4 changes: 3 additions & 1 deletion builder/ghc-for-component-wrapper.nix
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,11 @@ let
passthru = {
inherit (ghc) version meta;
};
propagatedBuildInputs = configFiles.libDeps;
nativeBuildInputs = [ghc];
} (''
configFiles=$(mktemp -d)
mkdir -p $out/configFiles
configFiles=$out/configFiles
${configFiles.script}
wrappedGhc=$out
${script}
Expand Down
20 changes: 13 additions & 7 deletions builder/haddock-builder.nix
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ let
finalConfigureFlags = lib.concatStringsSep " " (
[ "--prefix=${componentDrv}"
"${haskellLib.componentTarget componentId}"
"$(cat ${docsConfigFiles}/configure-flags)"
"$(cat $configFiles/configure-flags)"
]
++ commonConfigureFlags
++ lib.optional doHaddock' " --docdir=${docdir "$doc"}");
Expand All @@ -66,7 +66,7 @@ let
configFiles = docsConfigFiles;

# The directory containing the haddock documentation.
haddockDir = if doHaddock' then "${docdir drv.doc}/html" else null;
haddockDir = lib.const (if doHaddock' then "${docdir drv.doc}/html" else null);
};

# `out` contains the `package.conf.d` files used for building the
Expand All @@ -75,16 +75,22 @@ let
outputs = ["out"]
++ lib.optional doHaddock' "doc";

propagatedBuildInputs = builtins.concatLists pkgconfig;
propagatedBuildInputs =
builtins.concatLists pkgconfig
++ configFiles.libDeps;

buildInputs = component.libs
++ map (d: d.components.library.haddock or d) component.depends;
buildInputs = component.libs;

nativeBuildInputs =
[ shellWrappers buildPackages.removeReferencesTo ]
[ ghc buildPackages.removeReferencesTo ]
++ componentDrv.executableToolDepends;

configurePhase = ''
configFiles=$(mktemp -d)
${docsConfigFiles.script}
wrappedGhc=$(mktemp -d)
${shellWrappers.script}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now we're composing all these shell scripts together, maybe we should actually call them and pass the locations to write to as arguments rather than (effectively) sourcing them and passing arguments through variables? Not sure, just seems a little more explicit.

PATH=$wrappedGhc/bin:$PATH
runHook preConfigure
echo Configure flags:
printf "%q " ${finalConfigureFlags}
Expand Down Expand Up @@ -145,7 +151,7 @@ let
# working hyper links.
pkg=$(basename "$i")
sed -e "s|haddock-interfaces:.*|haddock-interfaces: $docdir/html/${componentId.cname}.haddock|" -e "s|haddock-html:.*|haddock-html: $docdir/html/|" "$i" > "$pkg"
${ghc.targetPrefix}ghc-pkg -v0 --package-db ${docsConfigFiles}/${configFiles.packageCfgDir} -f $out/package.conf.d register "$pkg"
${ghc.targetPrefix}ghc-pkg -v0 --package-db $configFiles/${configFiles.packageCfgDir} -f $out/package.conf.d register "$pkg"
done

ln -s ${componentDrv}/exactDep $out/exactDep
Expand Down
6 changes: 4 additions & 2 deletions builder/hspkg-builder.nix
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,17 @@ let

setup = if package.buildType == "Simple"
then ghc.defaultSetupFor package.identifier.name
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not that it makes much difference, but it sort of feels like I'd expect the setup exectuable to be associated with a cabal derivation rather than a GHC derivation.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would allow the project to select a newer or older version of Cabal the library if we put this on cabal-install (since haskell.nix could pick the default setup from the cabal-install passed to the project).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that good or bad? I think probably good?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes good. I'll try to set this up. I think we might need to keep the version of default setup built with Cabal from GHC on the ghc derivation (probably makes sense to have it there and for booting I think it might be necessary).

else setup-builder {
else setup-builder ({
component = components.setup // {
depends = config.setup-depends ++ components.setup.depends ++ package.setup-depends;
extraSrcFiles = components.setup.extraSrcFiles ++ [ "Setup.hs" "Setup.lhs" ];
pkgconfig = if components ? library then components.library.pkgconfig or [] else [];
};
inherit package name src flags revision patches defaultSetupSrc;
inherit (pkg) preUnpack postUnpack;
};
} // lib.optionalAttrs (package.buildType != "Custom") {
nonReinstallablePkgs = ["base" "Cabal"];
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

comment?

});

buildComp = allComponent: componentId: component: comp-builder {
inherit allComponent componentId component package name src flags setup cabalFile cabal-generator patches revision
Expand Down
3 changes: 2 additions & 1 deletion builder/make-config-files.nix
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ let
echo "allow-older: ${identifier.name}:*" >> $configFiles/cabal.config
''}

for p in $propagatedBuildInputs; do
for p in "''${pkgsHostTarget[@]}"; do
if [ -e $p/envDep ]; then
cat $p/envDep >> $configFiles/ghc-environment
fi
Expand Down Expand Up @@ -203,6 +203,7 @@ let
'';
drv = runCommand "${ghc.targetPrefix}${fullName}-config" {
nativeBuildInputs = [ghc];
propagatedBuildInputs = libDeps;
} (''
mkdir -p $out
configFiles=$out
Expand Down
2 changes: 1 addition & 1 deletion builder/setup-builder.nix
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ let
if [ -f $f ]; then
echo Compiling package $f
ghc $f -threaded ${if includeGhcPackage then "-package ghc " else ""
}-package-db $configFiles/${configFiles.packageCfgDir} --make -o ./Setup
}-package-env $configFiles/ghc-environment --make -o ./Setup
fi
done
[ -f ./Setup ] || (echo Failed to build Setup && exit 1)
Expand Down
6 changes: 3 additions & 3 deletions builder/shell-for.nix
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ let
# hoogle.nix expects.
docPackage = p: lib.getOutput "doc" p // {
pname = p.identifier.name;
haddockDir = lib.const p.haddockDir;
haddockDir = p.haddockDir;
};
in hoogleLocal ({
packages = map docPackage (haskellLib.flatLibDepends component);
Expand All @@ -149,7 +149,7 @@ in

buildInputs = systemInputs
++ mkDrvArgs.buildInputs or [];
nativeBuildInputs = [ ghcEnv ]
nativeBuildInputs = [ ghcEnv.drv ]
++ nativeBuildInputs
++ mkDrvArgs.nativeBuildInputs or []
++ lib.attrValues (buildPackages.haskell-nix.tools' evalPackages compiler.nix-name tools)
Expand Down Expand Up @@ -178,7 +178,7 @@ in

# This helps tools like `ghcide` (that use the ghc api) to find
# the correct global package DB.
NIX_GHC_LIBDIR = ghcEnv + "/" + configFiles.libDir;
NIX_GHC_LIBDIR = ghcEnv.drv + "/" + configFiles.libDir;

passthru = (mkDrvArgs.passthru or {}) // {
ghc = ghcEnv;
Expand Down