Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix --offline flag #8676

Merged
merged 20 commits into from
Mar 9, 2023
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@

{
// Automatically created by phoityne-vscode extension.

"version": "2.0.0",
"presentation": {
"reveal": "always",
"panel": "new"
},
"tasks": [
{
// F7
"group": {
"kind": "build",
"isDefault": true
},
"label": "haskell build",
"type": "shell",
//"command": "cabal configure && cabal build"
"command": "stack build"
},
{
// F6
"group": "build",
"type": "shell",
"label": "haskell clean & build",
//"command": "cabal clean && cabal configure && cabal build"
"command": "stack clean && stack build"
//"command": "stack clean ; stack build" // for powershell
},
{
// F8
"group": {
"kind": "test",
"isDefault": true
},
"type": "shell",
"label": "haskell test",
//"command": "cabal test"
"command": "stack test"
},
{
// F6
"isBackground": true,
"type": "shell",
"label": "haskell watch",
"command": "stack build --test --no-run-tests --file-watch"
}
]
}
85 changes: 60 additions & 25 deletions cabal-install/src/Distribution/Client/ProjectBuilding.hs
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,15 @@ import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as LBS
import qualified Data.ByteString.Lazy.Char8 as LBS.Char8

import qualified Text.PrettyPrint as Disp

import Control.Exception (Handler (..), SomeAsyncException, assert, catches, handle)
import System.Directory (canonicalizePath, createDirectoryIfMissing, doesDirectoryExist, doesFileExist, removeFile, renameDirectory)
import System.FilePath (dropDrive, makeRelative, normalise, takeDirectory, (<.>), (</>))
import System.IO (IOMode (AppendMode), Handle, withFile)

import Distribution.Compat.Directory (listDirectory)
import Distribution.Simple.Flag (fromFlagOrDefault)


------------------------------------------------------------------------------
Expand Down Expand Up @@ -559,6 +562,7 @@ invalidatePackageRegFileMonitor PackageFileMonitor{pkgFileMonitorReg} =
-- It requires the 'BuildStatusMap' gathered by 'rebuildTargetsDryRun'.
--
rebuildTargets :: Verbosity
-> ProjectConfig
-> DistDirLayout
-> StoreDirLayout
-> ElaboratedInstallPlan
Expand All @@ -567,6 +571,9 @@ rebuildTargets :: Verbosity
-> BuildTimeSettings
-> IO BuildOutcomes
rebuildTargets verbosity
ProjectConfig {
projectConfigBuildOnly = config
}
distDirLayout@DistDirLayout{..}
storeDirLayout
installPlan
Expand All @@ -579,7 +586,6 @@ rebuildTargets verbosity
buildSettingNumJobs,
buildSettingKeepGoing
} = do

-- Concurrency control: create the job controller and concurrency limits
-- for downloading, building and installing.
jobControl <- if isParallelBuild
Expand All @@ -599,30 +605,33 @@ rebuildTargets verbosity
createDirectoryIfMissingVerbose verbosity True distTempDirectory
traverse_ (createPackageDBIfMissing verbosity compiler progdb) packageDBsToUse

-- Before traversing the install plan, preemptively find all packages that
-- will need to be downloaded and start downloading them.
asyncDownloadPackages verbosity withRepoCtx
installPlan pkgsBuildStatus $ \downloadMap ->

-- For each package in the plan, in dependency order, but in parallel...
InstallPlan.execute jobControl keepGoing
(BuildFailure Nothing . DependentFailed . packageId)
installPlan $ \pkg ->
--TODO: review exception handling
handle (\(e :: BuildFailure) -> return (Left e)) $ fmap Right $

let uid = installedUnitId pkg
pkgBuildStatus = Map.findWithDefault (error "rebuildTargets") uid pkgsBuildStatus in

rebuildTarget
verbosity
distDirLayout
storeDirLayout
buildSettings downloadMap
registerLock cacheLock
sharedPackageConfig
installPlan pkg
pkgBuildStatus
if fromFlagOrDefault False (projectConfigOfflineMode config) && not (null packagesToDownload) then
ulysses4ever marked this conversation as resolved.
Show resolved Hide resolved
return offlineError
else
-- Before traversing the install plan, preemptively find all packages that
-- will need to be downloaded and start downloading them.
asyncDownloadPackages verbosity withRepoCtx
installPlan pkgsBuildStatus $ \downloadMap ->

-- For each package in the plan, in dependency order, but in parallel...
InstallPlan.execute jobControl keepGoing
(BuildFailure Nothing . DependentFailed . packageId)
installPlan $ \pkg ->
--TODO: review exception handling
handle (\(e :: BuildFailure) -> return (Left e)) $ fmap Right $

let uid = installedUnitId pkg
pkgBuildStatus = Map.findWithDefault (error "rebuildTargets") uid pkgsBuildStatus in

rebuildTarget
verbosity
distDirLayout
storeDirLayout
buildSettings downloadMap
registerLock cacheLock
sharedPackageConfig
installPlan pkg
pkgBuildStatus
where
isParallelBuild = buildSettingNumJobs >= 2
keepGoing = buildSettingKeepGoing
Expand All @@ -637,6 +646,32 @@ rebuildTargets verbosity
, elabSetupPackageDBStack elab ]
]

offlineError :: BuildOutcomes
offlineError = Map.fromList . map makeBuildOutcome $ packagesToDownload
where
makeBuildOutcome :: ElaboratedConfiguredPackage -> (UnitId, BuildOutcome)
makeBuildOutcome ElaboratedConfiguredPackage {
elabUnitId,
elabPkgSourceId = PackageIdentifier { pkgName, pkgVersion }
} = (elabUnitId, Left (BuildFailure {
buildFailureLogFile = Nothing,
buildFailureReason = DownloadFailed . error $ makeError pkgName pkgVersion
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it shouldn't involve error: it's not an exceptional situation like a network failure, or something else out of our control, it's we who are refusing to do something because of the flag -- it's no exception! So, I think, it should be DependentFailed rather than DownloadFailed. Perhaps someone has another opinion.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DependentFailed needs another package as a parameter this looks like it's used to say that the package that is failing has a dependency that failed. I need to say that this specific package failed while also not producing an exception. Nothing else in the BuildFailureReason data type looked like it suited the problem so I added a GracefulFailure option to BuildFailureReason and it takes a string that will be printed to the terminal without any extra text or callstack, I think anything more specific would clutter up the data type more than is needed.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Full disclosure: I'm no expert in Cabal error types. But, I thought that DependentFailed would work quite all right for this use case. Imagine you depend on 10 packages and 9 of them are in the cache. Then, you may want to know which one wasn't there. On the other hand, if there're two dependencies that are not in the cache then reporting just one is not so nice, perhaps. Although, I could live with reporting one that happened to be the first to be required during the build…

That said, I don't have a strong preference. I'm just not a big fan of changing data types that sound fundamental. Cabal is a library and this would be a breaking change...

}))
makeError :: PackageName -> Version -> String
makeError n v = "--offline was specified, could not download package "
ulysses4ever marked this conversation as resolved.
Show resolved Hide resolved
++ unPackageName n
++ " version " ++ Disp.render (pretty v)

packagesToDownload :: [ElaboratedConfiguredPackage]
packagesToDownload = [elab | InstallPlan.Configured elab <- InstallPlan.reverseTopologicalOrder installPlan,
isRemote $ elabPkgSourceLocation elab]
where
isRemote :: PackageLocation a -> Bool
isRemote (RemoteTarballPackage _ _) = True
isRemote (RepoTarballPackage {}) = True
isRemote (RemoteSourceRepoPackage _ _) = True
isRemote _ = False


-- | Create a package DB if it does not currently exist. Note that this action
-- is /not/ safe to run concurrently.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ runProjectBuildPhase verbosity
ProjectBaseContext{..} ProjectBuildContext {..} =
fmap (Map.union (previousBuildOutcomes pkgsBuildStatus)) $
rebuildTargets verbosity
projectConfig
distDirLayout
(cabalStoreDirLayout cabalDirLayout)
elaboratedPlanToExecute
Expand Down
3 changes: 2 additions & 1 deletion cabal-install/tests/IntegrationTests2.hs
Original file line number Diff line number Diff line change
Expand Up @@ -1736,7 +1736,7 @@ planProject testdir cliConfig = do
elaboratedShared)

executePlan :: PlanDetails -> IO (ElaboratedInstallPlan, BuildOutcomes)
executePlan ((distDirLayout, cabalDirLayout, _, _, buildSettings),
executePlan ((distDirLayout, cabalDirLayout, config, _, buildSettings),
elaboratedPlan,
elaboratedShared) = do

Expand All @@ -1762,6 +1762,7 @@ executePlan ((distDirLayout, cabalDirLayout, _, _, buildSettings),

buildOutcomes <-
rebuildTargets verbosity
config
distDirLayout
(cabalStoreDirLayout cabalDirLayout)
elaboratedPlan''
Expand Down
Binary file not shown.
4 changes: 4 additions & 0 deletions cabal-testsuite/PackageTests/OfflineFlag/Main.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import P (p)

main :: IO ()
main = print p
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
packages: ./main.cabal
19 changes: 19 additions & 0 deletions cabal-testsuite/PackageTests/OfflineFlag/main.cabal
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
cabal-version: 3.0
name: current
version: 0.1.0.0
license: MIT
author: Colton Clemmer
maintainer: coltonclemmerdev@gmail.com
-- copyright:
build-type: Simple
extra-doc-files: CHANGELOG.md
-- extra-source-files:

common warnings
ghc-options: -Wall

executable current
import: warnings
main-is: Main.hs
build-depends: base, remote
default-language: Haskell2010
25 changes: 25 additions & 0 deletions cabal-testsuite/PackageTests/OfflineFlag/offlineFlag.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# cabal v2-update
Downloading the latest package list from test-local-repo
# cabal v2-build
Resolving dependencies...
Build profile: -w ghc-<GHCVER> -O1
In order, the following will be built:
- remote-0.1.0.0 (lib) (requires build)
- current-0.1.0.0 (exe:current) (first run)
cabal: --offline was specified, could not download package remote version 0.1.0.0
CallStack (from HasCallStack):
ulysses4ever marked this conversation as resolved.
Show resolved Hide resolved
<CABAL_ERROR>
# cabal v2-build
Build profile: -w ghc-<GHCVER> -O1
In order, the following will be built:
- remote-0.1.0.0 (lib) (requires build)
- current-0.1.0.0 (exe:current) (first run)
Configuring library for remote-0.1.0.0..
Preprocessing library for remote-0.1.0.0..
Building library for remote-0.1.0.0..
Installing library in <PATH>
Configuring executable 'current' for current-0.1.0.0..
Preprocessing executable 'current' for current-0.1.0.0..
Building executable 'current' for current-0.1.0.0..
# cabal v2-build
Up to date
11 changes: 11 additions & 0 deletions cabal-testsuite/PackageTests/OfflineFlag/offlineFlag.test.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import Test.Cabal.Prelude

main = withShorterPathForNewBuildStore $ \storeDir ->
cabalTest $ do
skipUnlessGhcVersion ">= 8.1"
skipIfWindows
withProjectFile "cabal.repo.project" $ do
withRepo "repo" $ do
fails $ cabalG ["--store-dir=" ++ storeDir] "v2-build" ["current", "--offline"]
cabalG ["--store-dir=" ++ storeDir] "v2-build" ["current"]
cabalG ["--store-dir=" ++ storeDir] "v2-build" ["current", "--offline"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module P (p) where

p = "Foo"
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
cabal-version: 3.0
name: remote
version: 0.1.0.0
license: MIT
author: Colton Clemmer
maintainer: coltonclemmerdev@gmail.com
build-type: Simple
extra-doc-files: CHANGELOG.md

library
build-depends: base
exposed-modules: P
default-language: Haskell2010
1 change: 1 addition & 0 deletions cabal-testsuite/src/Test/Cabal/OutputNormalizer.hs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ normalizeOutput nenv =
-- Install path frequently has architecture specific elements, so
-- nub it out
. resub "Installing (.+) in .+" "Installing \\1 in <PATH>"
. resub " error, called at .*" "<CABAL_ERROR>"
ulysses4ever marked this conversation as resolved.
Show resolved Hide resolved
-- Things that look like libraries
. resub "libHS[A-Za-z0-9.-]+\\.(so|dll|a|dynlib)" "<LIBRARY>"
-- look for PackageHash directories
Expand Down
8 changes: 8 additions & 0 deletions changelog.d/pr-8676
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
synopsis: Adds functionality for the --offline flag with the "build" command.
packages: cabal-install
prs: #8676

description: {
The --offline flag previously created in #2578 but was only implemented for the install command even thought the flag didn't throw an error whenever the build command was run. This PR adds functionality for the --offline flag with the build command.
Additionally there is a new PackageTest for the flag using the build command.
}