Skip to content

Commit

Permalink
Various cabal-testsuite improvements (haskell#10225)
Browse files Browse the repository at this point in the history
* Improve bat scripts for CCompilerOverride

* Ensure Windows tests can cleanup the temp directory

* Implement `flaky` combinator

* Remove outdated tests

* Remove broken tests

These tests were testing for messages that were removed with the `cabal check` rework.

* Make `skip` and `broken` messages uniform

* Mark flaky tests

* Re-enable DeterministicTrivial

* Fix MacOS canonical paths

* Extend cabal-testsuite readme with `flaky`

* Skip non-terminating tests in Windows CI

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
  • Loading branch information
jasagredo and mergify[bot] authored Sep 2, 2024
1 parent 474d698 commit 39b6924
Show file tree
Hide file tree
Showing 82 changed files with 365 additions and 492 deletions.
23 changes: 19 additions & 4 deletions Cabal-tests/lib/Test/Utils/TempTestDir.hs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@

module Test.Utils.TempTestDir
( withTestDir
, withTestDir'
, removeDirectoryRecursiveHack
) where

import Distribution.Compat.Internal.TempFile (createTempDirectory)
import Distribution.Simple.Utils (warn)
import Distribution.Simple.Utils (warn, TempFileOptions (..), defaultTempFileOptions)
import Distribution.Verbosity

import Control.Concurrent (threadDelay)
Expand All @@ -23,12 +24,26 @@ import qualified System.Info (os)
-- | Much like 'withTemporaryDirectory' but with a number of hacks to make
-- sure on windows that we can clean up the directory at the end.
withTestDir :: (MonadIO m, MonadMask m) => Verbosity -> String -> (FilePath -> m a) -> m a
withTestDir verbosity template action = do
systmpdir <- liftIO getTemporaryDirectory
withTestDir verbosity template action = withTestDir' verbosity defaultTempFileOptions template action

withTestDir' :: (MonadIO m, MonadMask m) => Verbosity -> TempFileOptions -> String -> (FilePath -> m a) -> m a
withTestDir' verbosity tempFileOpts template action = do
systmpdir <-
-- MacOS returns /var/folders/... which is a symlink (/var -> /private/var),
-- so the test-suite struggles to make the build cwd-agnostic in particular
-- for the ShowBuildInfo tests. This canonicalizePath call makes it
-- /private/var/folders/... which will work.
liftIO $ canonicalizePath =<< getTemporaryDirectory
bracket
( do { tmpRelDir <- liftIO $ createTempDirectory systmpdir template
; return $ systmpdir </> tmpRelDir } )
(liftIO . removeDirectoryRecursiveHack verbosity)
(liftIO
-- This ensures that the temp files are not deleted at the end of the test.
-- It replicates the behavior of @withTempDirectoryEx@.
. when (not (optKeepTempFiles tempFileOpts))
-- This is the bit that helps with Windows deleting all files.
. removeDirectoryRecursiveHack verbosity
)
action

-- | On Windows, file locks held by programs we run (in this case VCSs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import Test.Cabal.Prelude

main = cabalTest $ do
skipUnlessGhcVersion ">= 8.1"
expectBrokenIf isWindows 10191 $ withProjectFile "cabal.internal.project" $ do
expectBrokenIfWindowsCI 10191 $ withProjectFile "cabal.internal.project" $ do
cabal "v2-build" ["exe"]
withPlan $ do
r <- runPlanExe' "I" "exe" []
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import Test.Cabal.Prelude
main = setupAndCabalTest $ do
skipUnlessGhcVersion ">= 8.1"
ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
expectBrokenIf ghc 7987 $ do
expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $ do
withPackageDb $ do
withDirectory "mylib" $ setup_install_with_docs ["--ipid", "mylib-0.1.0.0"]
withDirectory "mysql" $ setup_install_with_docs ["--ipid", "mysql-0.1.0.0"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ import Test.Cabal.Prelude
main = setupTest $ do
-- No cabal test because per-component is broken with it
skipUnlessGhcVersion ">= 8.1"
ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
expectBrokenIf ghc 7987 $
expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $
withPackageDb $
withDirectory "Includes2" $ do
let setup_install' args = setup_install_with_docs args
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import Test.Cabal.Prelude

main = cabalTest $ do
ghcVer <- isGhcVersion ">= 9.10"
skipUnlessGhcVersion ">= 8.1"
ghcVer <- isGhcVersion ">= 9.10"
skipIf "Windows + 9.10.1 (#10191)" (isWindows && ghcVer)
withProjectFile "cabal.external.project" $ do
cabal "v2-build" ["exe"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import Test.Cabal.Prelude

main = cabalTest $ do
skipUnlessGhcVersion ">= 8.1"
expectBrokenIf isWindows 10191 $ withProjectFile "cabal.internal.project" $ do
expectBrokenIfWindowsCI 10191 $ withProjectFile "cabal.internal.project" $ do
cabal "v2-build" ["exe"]
withPlan $ do
r <- runPlanExe' "I" "exe" []
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ import Data.List
import qualified Data.Char as Char
main = setupAndCabalTest $ do
skipUnlessGhcVersion ">= 8.1"
ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
expectBrokenIf ghc 7987 $
expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $
withPackageDb $ do
containers_id <- getIPID "containers"
withDirectory "repo/sigs-0.1.0.0" $ setup_install_with_docs ["--ipid", "sigs-0.1.0.0"]
Expand All @@ -21,4 +20,3 @@ main = setupAndCabalTest $ do
withDirectory "repo/exe-0.1.0.0" $ do
setup_install []
runExe' "exe" [] >>= assertOutputContains "fromList [(0,2),(2,4)]"

2 changes: 1 addition & 1 deletion cabal-testsuite/PackageTests/Backpack/T4754/setup.test.hs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude
main = setupAndCabalTest $ do
skipUnlessGhcVersion ">= 8.1"
skipUnless "no profiling libs" =<< hasProfiledLibraries
skipIfNoProfiledLibraries
setup "configure" ["--enable-profiling"]
setup "build" []
2 changes: 1 addition & 1 deletion cabal-testsuite/PackageTests/Backpack/T6385/cabal.test.hs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude
main =
cabalTest $ expectBrokenIf isWindows 10191 $ withShorterPathForNewBuildStore $ do
cabalTest $ expectBrokenIfWindows 10191 $ withShorterPathForNewBuildStore $ do
skipUnlessGhcVersion ">= 8.1"
withRepo "repo" $ do
cabal "v2-build" ["T6385"]
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
@echo OFF

where /q clang.exe

IF %ERRORLEVEL% EQU 0 (
call clang.exe -DNOERROR6 %*
EXIT /B %ERRORLEVEL%
)

ECHO "Cannot find C compiler"
EXIT /B 1
REM replace the libdir with the path to the compiler
FOR /f "delims=" %%A in ('call ghc.exe --print-libdir') do set "var=%%A"
setlocal EnableDelayedExpansion
CALL !var:lib=mingw\bin\clang.exe! -DNOERROR6 %*
EXIT /B %ERRORLEVEL%
14 changes: 5 additions & 9 deletions cabal-testsuite/PackageTests/CCompilerOverride/custom-cc.bat
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
@echo OFF

where /q gcc.exe

IF %ERRORLEVEL% EQU 0 (
call gcc.exe -DNOERROR6 %*
EXIT /B %ERRORLEVEL%
)

ECHO "Cannot find C compiler"
EXIT /B 1
REM replace the libdir with the path to the compiler
FOR /f "delims=" %%A in ('call ghc.exe --print-libdir') do set "var=%%A"
setlocal EnableDelayedExpansion
CALL !var:lib=mingw\bin\gcc.exe! -DNOERROR6 %*
EXIT /B %ERRORLEVEL%
2 changes: 1 addition & 1 deletion cabal-testsuite/PackageTests/CmmSourcesDyn/setup.test.hs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import Test.Cabal.Prelude

main = setupTest $ do
skipIf "ghc < 7.8" =<< isGhcVersion "< 7.8"
skipIfGhcVersion "< 7.8"
setup "configure" []
res <- setup' "build" []
assertOutputContains "= Post common block elimination =" res
1 change: 0 additions & 1 deletion cabal-testsuite/PackageTests/CustomDep/cabal.project

This file was deleted.

10 changes: 0 additions & 10 deletions cabal-testsuite/PackageTests/CustomDep/cabal.test.hs

This file was deleted.

2 changes: 0 additions & 2 deletions cabal-testsuite/PackageTests/CustomDep/client/B.hs

This file was deleted.

2 changes: 0 additions & 2 deletions cabal-testsuite/PackageTests/CustomDep/client/Setup.hs

This file was deleted.

12 changes: 0 additions & 12 deletions cabal-testsuite/PackageTests/CustomDep/client/client.cabal

This file was deleted.

1 change: 0 additions & 1 deletion cabal-testsuite/PackageTests/CustomDep/custom/A.hs

This file was deleted.

2 changes: 0 additions & 2 deletions cabal-testsuite/PackageTests/CustomDep/custom/Setup.hs

This file was deleted.

15 changes: 0 additions & 15 deletions cabal-testsuite/PackageTests/CustomDep/custom/custom.cabal

This file was deleted.

1 change: 0 additions & 1 deletion cabal-testsuite/PackageTests/CustomPlain/A.hs

This file was deleted.

3 changes: 0 additions & 3 deletions cabal-testsuite/PackageTests/CustomPlain/Setup.hs

This file was deleted.

1 change: 0 additions & 1 deletion cabal-testsuite/PackageTests/CustomPlain/cabal.project

This file was deleted.

11 changes: 0 additions & 11 deletions cabal-testsuite/PackageTests/CustomPlain/cabal.test.hs

This file was deleted.

11 changes: 0 additions & 11 deletions cabal-testsuite/PackageTests/CustomPlain/plain.cabal

This file was deleted.

7 changes: 0 additions & 7 deletions cabal-testsuite/PackageTests/CustomPlain/setup.cabal.out

This file was deleted.

6 changes: 0 additions & 6 deletions cabal-testsuite/PackageTests/CustomPlain/setup.out

This file was deleted.

4 changes: 0 additions & 4 deletions cabal-testsuite/PackageTests/CustomPlain/setup.test.hs

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import Test.Cabal.Prelude

main = do
cabalTest . expectBrokenIf isWindows 10179 . recordMode DoNotRecord $ do
cabalTest . expectBrokenIfWindows 10179 . recordMode DoNotRecord $ do
cwd <- fmap testCurrentDir getTestEnv
testInvokedWithBuildRunner cwd "test" []
testInvokedWithBuildRunner cwd "run" ["ghcjs-exe"]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude

main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
env <- getTestEnv
let cwd = testCurrentDir env
ghc_path <- programPathM ghcProgram
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude

main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
env <- getTestEnv
let cwd = testCurrentDir env
ghc_path <- programPathM ghcProgram
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude

main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
env <- getTestEnv
let cwd = testCurrentDir env
ghc_path <- programPathM ghcProgram
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude

main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
withSymlink "bin/ghc" "ghc" $ do
env <- getTestEnv
let cwd = testCurrentDir env
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude

main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
withSymlink "bin/ghc-7.10" "ghc" $ do
env <- getTestEnv
let cwd = testCurrentDir env
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Test.Cabal.Prelude

main = setupAndCabalTest $ expectBrokenIf isWindows 10179 $ do
main = setupAndCabalTest $ expectBrokenIfWindows 10179 $ do
withSymlink "bin/ghc-7.10" "ghc" $ do
env <- getTestEnv
let cwd = testCurrentDir env
Expand Down

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

Loading

0 comments on commit 39b6924

Please sign in to comment.