Skip to content

Commit 03e8346

Browse files
andreabediniandreabedini
committed
Add separate cache for getPkgConfigDb
Querying pkg-config for the version of every module can be a very expensive operation on some systems. This change adds a separate, per-project, cache for PkgConfigDB; reducing the cost from "every plan change" to "every pkg-config-db change per project". The cache key is composed by the pkg-config configured program and the list of directories reported by pkg-config's pc_path variable. A notice is presented to the user when refreshing the PkgConfigDB.
1 parent c97092f commit 03e8346

File tree

148 files changed

+265
-48
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

148 files changed

+265
-48
lines changed

‎cabal-install/src/Distribution/Client/ProjectConfig.hs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ module Distribution.Client.ProjectConfig
5050
, resolveSolverSettings
5151
, BuildTimeSettings (..)
5252
, resolveBuildTimeSettings
53+
, resolveProgramDb
5354

5455
-- * Checking configuration
5556
, checkBadPerPackageCompilerPaths
@@ -154,6 +155,14 @@ import Distribution.Simple.InstallDirs
154155
)
155156
import Distribution.Simple.Program
156157
( ConfiguredProgram (..)
158+
, ProgramSearchPathEntry (..)
159+
)
160+
import Distribution.Simple.Program.Db
161+
( ProgramDb
162+
, defaultProgramDb
163+
, modifyProgramSearchPath
164+
, userSpecifyArgss
165+
, userSpecifyPaths
157166
)
158167
import Distribution.Simple.Setup
159168
( Flag (Flag)
@@ -496,6 +505,19 @@ resolveBuildTimeSettings
496505
| isParallelBuild buildSettingNumJobs = False
497506
| otherwise = False
498507

508+
-- | ProgramDb with directly user specified paths
509+
resolveProgramDb :: ProjectConfig -> ProgramDb
510+
resolveProgramDb ProjectConfig{projectConfigLocalPackages = PackageConfig{..}} =
511+
userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
512+
. userSpecifyArgss (Map.toList (getMapMappend packageConfigProgramArgs))
513+
. modifyProgramSearchPath
514+
( ++
515+
[ ProgramSearchPathDir dir
516+
| dir <- fromNubList packageConfigProgramPathExtra
517+
]
518+
)
519+
$ defaultProgramDb
520+
499521
---------------------------------------------
500522
-- Reading and writing project config files
501523
--

‎cabal-install/src/Distribution/Client/ProjectPlanning.hs

Lines changed: 31 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ configureCompiler
481481
DistDirLayout
482482
{ distProjectCacheFile
483483
}
484-
ProjectConfig
484+
projectConfig@ProjectConfig
485485
{ projectConfigShared =
486486
ProjectConfigShared
487487
{ projectConfigHcFlavor
@@ -531,15 +531,7 @@ configureCompiler
531531
hcFlavor = flagToMaybe projectConfigHcFlavor
532532
hcPath = flagToMaybe projectConfigHcPath
533533
hcPkg = flagToMaybe projectConfigHcPkg
534-
progdb =
535-
userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
536-
. modifyProgramSearchPath
537-
( [ ProgramSearchPathDir dir
538-
| dir <- fromNubList packageConfigProgramPathExtra
539-
]
540-
++
541-
)
542-
$ defaultProgramDb
534+
progdb = resolveProgramDb projectConfig
543535

544536
------------------------------------------------------------------------------
545537

@@ -588,6 +580,9 @@ rebuildInstallPlan
588580
progsearchpath <- liftIO $ getSystemSearchPath
589581
let projectConfigMonitored = projectConfig{projectConfigBuildOnly = mempty}
590582

583+
let progdb = resolveProgramDb projectConfig
584+
pkgConfigDB <- getPkgConfigDb verbosity distDirLayout progdb
585+
591586
-- The overall improved plan is cached
592587
rerunIfChanged
593588
verbosity
@@ -607,11 +602,12 @@ rebuildInstallPlan
607602
)
608603
$ do
609604
compilerEtc <- phaseConfigureCompiler projectConfig
610-
_ <- phaseConfigurePrograms projectConfig compilerEtc
611-
(solverPlan, pkgConfigDB, totalIndexState, activeRepos) <-
605+
606+
(solverPlan, totalIndexState, activeRepos) <-
612607
phaseRunSolver
613608
projectConfig
614609
compilerEtc
610+
pkgConfigDB
615611
localPackages
616612
(fromMaybe mempty mbInstalledPackages)
617613
( elaboratedPlan
@@ -650,55 +646,37 @@ rebuildInstallPlan
650646
phaseConfigureCompiler
651647
:: ProjectConfig
652648
-> Rebuild (Compiler, Platform, ProgramDb)
653-
phaseConfigureCompiler = configureCompiler verbosity distDirLayout
649+
phaseConfigureCompiler projectConfig = do
650+
(compiler, platform, progdb) <- configureCompiler verbosity distDirLayout projectConfig
654651

655-
-- Configuring other programs.
656-
--
657-
-- Having configred the compiler, now we configure all the remaining
658-
-- programs. This is to check we can find them, and to monitor them for
659-
-- changes.
660-
--
661-
-- TODO: [required eventually] we don't actually do this yet.
662-
--
663-
-- We rely on the fact that the previous phase added the program config for
664-
-- all local packages, but that all the programs configured so far are the
665-
-- compiler program or related util programs.
666-
--
667-
phaseConfigurePrograms
668-
:: ProjectConfig
669-
-> (Compiler, Platform, ProgramDb)
670-
-> Rebuild ()
671-
phaseConfigurePrograms projectConfig (_, _, compilerprogdb) = do
672652
-- Users are allowed to specify program locations independently for
673653
-- each package (e.g. to use a particular version of a pre-processor
674654
-- for some packages). However they cannot do this for the compiler
675655
-- itself as that's just not going to work. So we check for this.
676656
liftIO $
677657
checkBadPerPackageCompilerPaths
678-
(configuredPrograms compilerprogdb)
658+
(configuredPrograms progdb)
679659
(getMapMappend (projectConfigSpecificPackage projectConfig))
680660

681-
-- TODO: [required eventually] find/configure other programs that the
682-
-- user specifies.
683-
684-
-- TODO: [required eventually] find/configure all build-tools
685-
-- but note that some of them may be built as part of the plan.
661+
return (compiler, platform, progdb)
686662

687663
-- Run the solver to get the initial install plan.
688664
-- This is expensive so we cache it independently.
689665
--
690666
phaseRunSolver
691667
:: ProjectConfig
692668
-> (Compiler, Platform, ProgramDb)
669+
-> PkgConfigDb
693670
-> [PackageSpecifier UnresolvedSourcePackage]
694671
-> InstalledPackageIndex
695-
-> Rebuild (SolverInstallPlan, PkgConfigDb, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
672+
-> Rebuild (SolverInstallPlan, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
696673
phaseRunSolver
697674
projectConfig@ProjectConfig
698675
{ projectConfigShared
699676
, projectConfigBuildOnly
700677
}
701678
(compiler, platform, progdb)
679+
pkgConfigDB
702680
localPackages
703681
installedPackages =
704682
rerunIfChanged
@@ -719,13 +697,13 @@ rebuildInstallPlan
719697
progdb
720698
platform
721699
corePackageDbs
700+
722701
(sourcePkgDb, tis, ar) <-
723702
getSourcePackages
724703
verbosity
725704
withRepoCtx
726705
(solverSettingIndexState solverSettings)
727706
(solverSettingActiveRepos solverSettings)
728-
pkgConfigDB <- getPkgConfigDb verbosity progdb
729707

730708
-- TODO: [code cleanup] it'd be better if the Compiler contained the
731709
-- ConfiguredPrograms that it needs, rather than relying on the progdb
@@ -750,7 +728,7 @@ rebuildInstallPlan
750728
Left msg -> do
751729
reportPlanningFailure projectConfig compiler platform localPackages
752730
dieWithException verbosity $ PhaseRunSolverErr msg
753-
Right plan -> return (plan, pkgConfigDB, tis, ar)
731+
Right plan -> return (plan, tis, ar)
754732
where
755733
corePackageDbs :: [PackageDB]
756734
corePackageDbs =
@@ -1036,13 +1014,21 @@ getSourcePackages verbosity withRepoCtx idxState activeRepos = do
10361014
$ repos
10371015
return sourcePkgDbWithTIS
10381016

1039-
getPkgConfigDb :: Verbosity -> ProgramDb -> Rebuild PkgConfigDb
1040-
getPkgConfigDb verbosity progdb = do
1017+
getPkgConfigDb :: Verbosity -> DistDirLayout -> ProgramDb -> Rebuild PkgConfigDb
1018+
getPkgConfigDb verbosity distDirLayout progdb = do
1019+
-- We find the pkg-config program and we ask it for the location of the pc files.
1020+
-- We then use these two values for the cache key
1021+
mpkgConfig <- liftIO $ needProgram verbosity pkgConfigProgram progdb
1022+
let mpkgConfigLoc = fst <$> mpkgConfig
10411023
dirs <- liftIO $ getPkgConfigDbDirs verbosity progdb
1042-
-- Just monitor the dirs so we'll notice new .pc files.
1043-
-- Alternatively we could monitor all the .pc files too.
1044-
traverse_ monitorDirectoryStatus dirs
1045-
liftIO $ readPkgConfigDb verbosity progdb
1024+
rerunIfChanged verbosity fileMonitorPkgConfigDb (mpkgConfigLoc, dirs) $ do
1025+
liftIO $ notice verbosity "Querying pkg-config database..."
1026+
-- Just monitor the dirs so we'll notice new .pc files.
1027+
-- Alternatively we could monitor all the .pc files too.
1028+
traverse_ monitorDirectoryStatus dirs
1029+
liftIO $ readPkgConfigDb verbosity progdb
1030+
where
1031+
fileMonitorPkgConfigDb = newFileMonitor $ distProjectCacheFile distDirLayout "pkg-config-db"
10461032

10471033
-- | Select the config values to monitor for changes package source hashes.
10481034
packageLocationsSignature

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external-target.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal-target.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:
@@ -18,8 +19,7 @@ Configuring library 'indef' instantiated with
1819
Data.Map = containers-<VERSION>:Data.Map
1920
for Includes3-0.1.0.0...
2021
Preprocessing library 'indef' for Includes3-0.1.0.0...
21-
Building library 'indef' instantiated with
22-
Data.Map = containers-<VERSION>:Data.Map
22+
Building library 'indef' instantiated with Data.Map = containers-<VERSION>:Data.Map
2323
for Includes3-0.1.0.0...
2424
Configuring executable 'exe' for Includes3-0.1.0.0...
2525
Preprocessing executable 'exe' for Includes3-0.1.0.0...

‎cabal-testsuite/PackageTests/Backpack/Includes3/cabal-repo.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Reexport2/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Error:
45
Problem with module re-exports:

‎cabal-testsuite/PackageTests/Backpack/T6385/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/bkpcabal01/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/BuildDeps/DepCycle/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Error:
45
Dependency cycle between the following components:

‎cabal-testsuite/PackageTests/BuildDeps/InternalLibrary1/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/BuildTargets/UseLocalPackage/use-local-version-of-package.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

‎cabal-testsuite/PackageTests/BuildTargets/UseLocalPackageForSetup/use-local-package-as-setup-dep.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Error: [Cabal-7107]
67
Could not resolve dependencies:

‎cabal-testsuite/PackageTests/BuildToolDepends/setup.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/BuildTools/External/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/BuildTools/Internal/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/CmmSources/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-run
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/CmmSourcesDyn/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-run
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/CmmSourcesExe/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-run
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/ConditionalAndImport/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-run
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

‎cabal-testsuite/PackageTests/ConfigFile/T8487/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O2
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Configure/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/CopyHie/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

‎cabal-testsuite/PackageTests/CustomSegfault/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/CustomWithoutCabal/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/CustomWithoutCabalDefaultMain/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Exec/cabal.out

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

0 commit comments

Comments
 (0)