From 3d0ae982b42722ab1ea567f4d109e3e0f4840e68 Mon Sep 17 00:00:00 2001 From: taoky Date: Wed, 3 Jan 2024 16:49:36 +0800 Subject: [PATCH] Upgrade to Debian 12 (#106) * Upgrade to Debian 12 - yum-sync: backports is no longer needed - stackage: reorganize to fully utilize Debian built Haskell packages - ghcup: use Debian built Haskell packages --- README.md | 6 +- base/Dockerfile.debian | 2 +- ghcup/Dockerfile | 2 +- ghcup/config | 117 ------------ ghcup/prepare.sh | 21 +-- stackage/Dockerfile | 3 +- stackage/README | 1 + stackage/config | 234 ----------------------- stackage/prepare.sh | 17 +- stackage/stackage.hs | 411 ++++++++++++++++++++++------------------- yum-sync/Dockerfile | 3 +- 11 files changed, 246 insertions(+), 571 deletions(-) delete mode 100644 ghcup/config create mode 100644 stackage/README delete mode 100644 stackage/config diff --git a/README.md b/README.md index e61f414..28dc6d8 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Notice: BIND_ADDRESS is only added for `curl` in freebsd-ports. Make sure that g [![ghcup](https://img.shields.io/docker/image-size/ustcmirror/ghcup/latest)](https://hub.docker.com/r/ustcmirror/ghcup "ghcup") [![ghcup](https://img.shields.io/docker/pulls/ustcmirror/ghcup)](https://hub.docker.com/r/ustcmirror/ghcup "ghcup") -ghcup does not have outstanding configuration options. See also [stackage](#stackage) on replacing the Hackage repository. +ghcup does not have outstanding configuration options. ### github-release @@ -413,9 +413,7 @@ ref: [![stackage](https://img.shields.io/docker/image-size/ustcmirror/stackage/latest)](https://hub.docker.com/r/ustcmirror/stackage "stackage") [![stackage](https://img.shields.io/docker/pulls/ustcmirror/stackage)](https://hub.docker.com/r/ustcmirror/stackage "stackage") -Stackage doesn't need to specify upstream, but this mirror use cabal to install necessary Haskell packages. Replacing default mirror of cabal with faster one will speed up building process. - -Read the [user guide](https://www.haskell.org/cabal/users-guide/installing-packages.html#repository-specification) before writing preferred mirror to `config` +Stackage doesn't need to specify upstream. ### tsumugu diff --git a/base/Dockerfile.debian b/base/Dockerfile.debian index 9b5c60d..4fe0c9e 100644 --- a/base/Dockerfile.debian +++ b/base/Dockerfile.debian @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim +FROM debian:bookworm-slim LABEL maintainer="Jian Zeng " \ org.ustcmirror.images=true ADD ["entry.sh", "savelog", "/usr/local/bin/"] diff --git a/ghcup/Dockerfile b/ghcup/Dockerfile index 12e40c3..ce7c4cc 100644 --- a/ghcup/Dockerfile +++ b/ghcup/Dockerfile @@ -1,4 +1,4 @@ FROM ustcmirror/base:debian MAINTAINER Kai Ma -ADD ["config", "sync.sh", "prepare.sh", "ghcupsync.hs", "ghcupsync.cabal", "/"] +ADD ["sync.sh", "prepare.sh", "ghcupsync.hs", "/"] RUN bash /prepare.sh && rm prepare.sh diff --git a/ghcup/config b/ghcup/config deleted file mode 100644 index c1cae28..0000000 --- a/ghcup/config +++ /dev/null @@ -1,117 +0,0 @@ -repository mirrors.ustc.edu.cn - url: http://mirrors.ustc.edu.cn/hackage/ - secure: True - - -remote-repo-cache: /root/.cabal/packages -world-file: /root/.cabal/world -extra-prog-path: /root/.cabal/bin -build-summary: /root/.cabal/logs/build.log -remote-build-reporting: anonymous -jobs: $ncpus - -haddock - -- keep-temp-files: False - -- hoogle: False - -- html: False - -- html-location: - -- executables: False - -- tests: False - -- benchmarks: False - -- foreign-libraries: False - -- all: - -- internal: False - -- css: - -- hyperlink-source: False - -- hscolour-css: - -- contents-location: - -install-dirs user - -- prefix: /root/.cabal - -- bindir: $prefix/bin - -- libdir: $prefix/lib - -- libsubdir: $abi/$libname - -- dynlibdir: $libdir/$abi - -- libexecdir: $prefix/libexec - -- libexecsubdir: $abi/$pkgid - -- datadir: $prefix/share - -- datasubdir: $abi/$pkgid - -- docdir: $datadir/doc/$abi/$pkgid - -- htmldir: $docdir/html - -- haddockdir: $htmldir - -- sysconfdir: $prefix/etc - -install-dirs global - -- prefix: /usr/local - -- bindir: $prefix/bin - -- libdir: $prefix/lib - -- libsubdir: $abi/$libname - -- dynlibdir: $libdir/$abi - -- libexecdir: $prefix/libexec - -- libexecsubdir: $abi/$pkgid - -- datadir: $prefix/share - -- datasubdir: $abi/$pkgid - -- docdir: $datadir/doc/$abi/$pkgid - -- htmldir: $docdir/html - -- haddockdir: $htmldir - -- sysconfdir: $prefix/etc - -program-locations - -- alex-location: - -- ar-location: - -- c2hs-location: - -- cpphs-location: - -- doctest-location: - -- gcc-location: - -- ghc-location: - -- ghc-pkg-location: - -- ghcjs-location: - -- ghcjs-pkg-location: - -- greencard-location: - -- haddock-location: - -- happy-location: - -- haskell-suite-location: - -- haskell-suite-pkg-location: - -- hmake-location: - -- hpc-location: - -- hsc2hs-location: - -- hscolour-location: - -- jhc-location: - -- ld-location: - -- lhc-location: - -- lhc-pkg-location: - -- pkg-config-location: - -- runghc-location: - -- strip-location: - -- tar-location: - -- uhc-location: - -program-default-options - -- alex-options: - -- ar-options: - -- c2hs-options: - -- cpphs-options: - -- doctest-options: - -- gcc-options: - -- ghc-options: - -- ghc-pkg-options: - -- ghcjs-options: - -- ghcjs-pkg-options: - -- greencard-options: - -- haddock-options: - -- happy-options: - -- haskell-suite-options: - -- haskell-suite-pkg-options: - -- hmake-options: - -- hpc-options: - -- hsc2hs-options: - -- hscolour-options: - -- jhc-options: - -- ld-options: - -- lhc-options: - -- lhc-pkg-options: - -- pkg-config-options: - -- runghc-options: - -- strip-options: - -- tar-options: - -- uhc-options: diff --git a/ghcup/prepare.sh b/ghcup/prepare.sh index 11575ce..61a2a75 100644 --- a/ghcup/prepare.sh +++ b/ghcup/prepare.sh @@ -1,19 +1,16 @@ #!/bin/bash set -e -mkdir /root/.cabal -mv config /root/.cabal/ +HASKELL_DEPS="ghc libghc-aeson-dev libghc-lens-dev libghc-lens-aeson-dev \ + libghc-network-uri-dev libghc-split-dev libghc-typed-process-dev \ + libghc-yaml-dev" +# Please use ldd to check the dependencies of compiled binary +HASKELL_KEEP="libyaml-0-2" apt-get update -apt-get install -y aria2 ca-certificates git wget xz-utils apt-utils build-essential curl libffi-dev libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5 libnuma-dev +apt-get install -y aria2 ca-certificates git $HASKELL_DEPS $HASKELL_KEEP -curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_MINIMAL=1 sh -source /root/.ghcup/env -ghcup install ghc 9.0.2 -ghcup install cabal 3.6.2.0 -ghcup set ghc 9.0.2 -cabal update -cabal build -O2 -cp $(cabal list-bin -O2 ghcupsync) / +# compile with Haskell language extensions +ghc -O2 -XScopedTypeVariables -XOverloadedStrings ghcupsync.hs -rm -rf /dist* /root/.cabal /root/.ghcup /root/.ghc && apt-get purge -y --auto-remove build-essential libffi-dev libgmp-dev libncurses-dev && rm -rf /var/lib/apt/lists/* +rm -rf /ghcupsync.* && apt-get purge -y --auto-remove $HASKELL_DEPS && rm -rf /var/lib/apt/lists/* diff --git a/stackage/Dockerfile b/stackage/Dockerfile index 7fab619..763913b 100644 --- a/stackage/Dockerfile +++ b/stackage/Dockerfile @@ -1,4 +1,5 @@ FROM ustcmirror/base:debian MAINTAINER Jiahao Li -ADD ["config", "sync.sh", "prepare.sh", "stackage.hs", "/"] +MAINTAINER Keyu Tao +ADD ["sync.sh", "prepare.sh", "stackage.hs", "/"] RUN bash /prepare.sh && rm prepare.sh diff --git a/stackage/README b/stackage/README new file mode 100644 index 0000000..14cdd50 --- /dev/null +++ b/stackage/README @@ -0,0 +1 @@ +本地调试可参考 stackage-stack 分支： diff --git a/stackage/config b/stackage/config deleted file mode 100644 index 7ae1067..0000000 --- a/stackage/config +++ /dev/null @@ -1,234 +0,0 @@ --- This is the configuration file for the 'cabal' command line tool. --- --- The available configuration options are listed below. --- Some of them have default values listed. --- --- Lines (like this one) beginning with '--' are comments. --- Be careful with spaces and indentation because they are --- used to indicate layout for nested sections. --- --- This config file was generated using the following versions --- of Cabal and cabal-install: --- Cabal library version: 3.0.1.0 --- cabal-install version: 3.0.0.0 - - -repository mirrors.ustc.edu.cn - url: http://mirrors.ustc.edu.cn/hackage/ - -- secure: True - -- root-keys: - -- key-threshold: 3 - --- default-user-config: --- require-sandbox: False --- ignore-sandbox: False --- ignore-expiry: False --- http-transport: --- nix: False -remote-repo-cache: /root/.cabal/packages --- local-repo: --- logs-dir: /root/.cabal/logs -world-file: /root/.cabal/world --- store-dir: --- verbose: 1 --- compiler: ghc --- cabal-file: --- with-compiler: --- with-hc-pkg: --- program-prefix: --- program-suffix: --- library-vanilla: True --- library-profiling: --- shared: --- static: --- executable-dynamic: False --- executable-static: False --- profiling: --- executable-profiling: --- profiling-detail: --- library-profiling-detail: --- optimization: True --- debug-info: False --- library-for-ghci: --- split-sections: False --- split-objs: False --- executable-stripping: --- library-stripping: --- configure-option: --- user-install: True --- package-db: --- flags: --- extra-include-dirs: --- deterministic: --- cid: --- extra-lib-dirs: --- extra-framework-dirs: -extra-prog-path: /root/.cabal/bin --- instantiate-with: --- tests: False --- coverage: False --- library-coverage: --- exact-configuration: False --- benchmarks: False --- relocatable: False --- response-files: --- allow-depending-on-private-libs: --- cabal-lib-version: --- constraint: --- preference: --- solver: modular --- allow-older: False --- allow-newer: False --- write-ghc-environment-files: --- documentation: False --- doc-index-file: $datadir/doc/$arch-$os-$compiler/index.html --- target-package-db: --- max-backjumps: 4000 --- reorder-goals: False --- count-conflicts: True --- minimize-conflict-set: False --- independent-goals: False --- shadow-installed-packages: False --- strong-flags: False --- allow-boot-library-installs: False --- reject-unconstrained-dependencies: none --- reinstall: False --- avoid-reinstalls: False --- force-reinstalls: False --- upgrade-dependencies: False --- index-state: --- root-cmd: --- symlink-bindir: -build-summary: /root/.cabal/logs/build.log --- build-log: -remote-build-reporting: anonymous --- report-planning-failure: False --- per-component: True --- one-shot: False --- run-tests: -jobs: $ncpus --- keep-going: False --- offline: False --- project-file: --- lib: False --- package-env: --- overwrite-policy: --- install-method: -installdir: /root/.cabal/bin --- username: --- password: --- password-command: --- builddir: - -haddock - -- keep-temp-files: False - -- hoogle: False - -- html: False - -- html-location: - -- executables: False - -- tests: False - -- benchmarks: False - -- foreign-libraries: False - -- all: - -- internal: False - -- css: - -- hyperlink-source: False - -- quickjump: False - -- hscolour-css: - -- contents-location: - -init - -- interactive: False - -- cabal-version: 1.10 - -- license: BSD3 - -- tests: - -- test-dir: - -- language: Haskell2010 - -- application-dir: - -- source-dir: - -install-dirs user - -- prefix: /root/.cabal - -- bindir: $prefix/bin - -- libdir: $prefix/lib - -- libsubdir: $abi/$libname - -- dynlibdir: $libdir/$abi - -- libexecdir: $prefix/libexec - -- libexecsubdir: $abi/$pkgid - -- datadir: $prefix/share - -- datasubdir: $abi/$pkgid - -- docdir: $datadir/doc/$abi/$pkgid - -- htmldir: $docdir/html - -- haddockdir: $htmldir - -- sysconfdir: $prefix/etc - -install-dirs global - -- prefix: /usr/local - -- bindir: $prefix/bin - -- libdir: $prefix/lib - -- libsubdir: $abi/$libname - -- dynlibdir: $libdir/$abi - -- libexecdir: $prefix/libexec - -- libexecsubdir: $abi/$pkgid - -- datadir: $prefix/share - -- datasubdir: $abi/$pkgid - -- docdir: $datadir/doc/$abi/$pkgid - -- htmldir: $docdir/html - -- haddockdir: $htmldir - -- sysconfdir: $prefix/etc - -program-locations - -- alex-location: - -- ar-location: - -- c2hs-location: - -- cpphs-location: - -- doctest-location: - -- gcc-location: - -- ghc-location: - -- ghc-pkg-location: - -- ghcjs-location: - -- ghcjs-pkg-location: - -- greencard-location: - -- haddock-location: - -- happy-location: - -- haskell-suite-location: - -- haskell-suite-pkg-location: - -- hmake-location: - -- hpc-location: - -- hsc2hs-location: - -- hscolour-location: - -- jhc-location: - -- ld-location: - -- pkg-config-location: - -- runghc-location: - -- strip-location: - -- tar-location: - -- uhc-location: - -program-default-options - -- alex-options: - -- ar-options: - -- c2hs-options: - -- cpphs-options: - -- doctest-options: - -- gcc-options: - -- ghc-options: - -- ghc-pkg-options: - -- ghcjs-options: - -- ghcjs-pkg-options: - -- greencard-options: - -- haddock-options: - -- happy-options: - -- haskell-suite-options: - -- haskell-suite-pkg-options: - -- hmake-options: - -- hpc-options: - -- hsc2hs-options: - -- hscolour-options: - -- jhc-options: - -- ld-options: - -- pkg-config-options: - -- runghc-options: - -- strip-options: - -- tar-options: - -- uhc-options: \ No newline at end of file diff --git a/stackage/prepare.sh b/stackage/prepare.sh index 5016bce..53fc91c 100755 --- a/stackage/prepare.sh +++ b/stackage/prepare.sh @@ -1,10 +1,13 @@ #!/bin/bash set -e + +HASKELL_DEPS="ghc libghc-yaml-dev libghc-process-extras-dev \ + libghc-aeson-dev libghc-split-dev" +# Please use ldd to check the dependencies of compiled binary +HASKELL_KEEP="libyaml-0-2" + apt-get update -apt-get install -y aria2 ca-certificates git wget libghc-bzlib-dev xz-utils musl-dev apt-utils ghc cabal-install haskell-platform libghc-cabal-dev -mkdir /root/.cabal -mv config /root/.cabal/ -cabal update -cabal install --lib yaml-0.11.8.0 process containers aeson-2.1.1.0 -ghc -O2 stackage.hs -package unordered-containers -package split -rm -rf /root/.cabal && apt-get purge -y --auto-remove libghc-bzlib-dev xz-utils musl-dev apt-utils ghc cabal-install haskell-platform libghc-cabal-dev && rm -rf /var/lib/apt/lists/* +apt-get install -y aria2 ca-certificates git $HASKELL_DEPS $HASKELL_KEEP +ghc -O2 stackage.hs + +rm -rf stackage.* && apt-get purge --auto-remove -y $HASKELL_DEPS && rm -rf /var/lib/apt/lists/* diff --git a/stackage/stackage.hs b/stackage/stackage.hs index 21988e1..ac43aa3 100644 --- a/stackage/stackage.hs +++ b/stackage/stackage.hs @@ -1,29 +1,30 @@ -{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +import Control.Monad (unless) +import Data.Aeson qualified as A +import Data.ByteString.Lazy qualified as BS +import Data.List.Split (splitOn) +import Data.Map.Strict qualified as M import Data.Yaml -import qualified Data.Aeson as A -import Data.Aeson.Types (ToJSON) -import Data.Maybe (isNothing, fromJust) -import qualified Data.Map.Strict as M -import qualified Data.ByteString.Lazy as BS -import qualified Data.Aeson.KeyMap as KM -import qualified Data.Text as T -import System.IO (readFile, writeFile) -import System.Exit (ExitCode(..)) -import System.Process (callProcess, readProcessWithExitCode) -import System.Directory -import System.Environment (getEnv) -import System.FilePath.Posix (FilePath, ()) -import Control.Exception (catch) -import Control.Monad (when, unless) -import Data.List.Split (splitOn) -import Text.Printf (printf) +import System.Directory (createDirectory, doesDirectoryExist, doesFileExist) +import System.Environment (getEnv) +import System.Exit (ExitCode (..)) +import System.FilePath.Posix (()) +import System.Process (callProcess, readProcessWithExitCode) +import Text.Printf (printf) type URL = String + type SHA1 = String + type SHA256 = String + type Platform = String + type Version = String + type Path = String -- @@ -31,229 +32,255 @@ type Path = String -- are for stack-setup-2.yaml parsing -- -data ResourceInfo = ResourceInfo { - version :: String, - url :: String, - contentLength :: Int, - sha1 :: SHA1, - sha256 :: SHA256 -} deriving (Show) - -data GhcJSInfo = GhcJSInfo { - source :: M.Map String ResourceInfo -} deriving (Show) - - -data StackSetup = StackSetup { - stack :: M.Map Platform (M.Map Version ResourceInfo), - sevenzexeInfo :: ResourceInfo, - sevenzdllInfo :: ResourceInfo, - portableGit :: ResourceInfo, - msys2 :: M.Map Platform ResourceInfo, - ghc :: M.Map Platform (M.Map Version ResourceInfo), - ghcjs :: GhcJSInfo -} deriving (Show) +data ResourceInfo = ResourceInfo + { version :: String, + url :: String, + contentLength :: Int, + sha1 :: SHA1, + sha256 :: SHA256 + } + deriving (Show) + +newtype GhcJSInfo = GhcJSInfo + { source :: M.Map String ResourceInfo + } + deriving (Show) + +data StackSetup = StackSetup + { stack :: M.Map Platform (M.Map Version ResourceInfo), + sevenzexeInfo :: ResourceInfo, + sevenzdllInfo :: ResourceInfo, + portableGit :: ResourceInfo, + msys2 :: M.Map Platform ResourceInfo, + ghc :: M.Map Platform (M.Map Version ResourceInfo), + ghcjs :: GhcJSInfo + } + deriving (Show) + +data GitHubReleases = GitHubReleases + { prerelease :: Bool, + urls :: [URL] + } + deriving (Show) instance FromJSON GhcJSInfo where - parseJSON = withObject "GhcJSInfo" $ \o -> do - source <- o .: "source" - return GhcJSInfo {..} + parseJSON = withObject "GhcJSInfo" $ \o -> do + source <- o .: "source" + return GhcJSInfo {..} instance ToJSON GhcJSInfo where - toJSON (GhcJSInfo source) = object - ["source" .= source] - + toJSON (GhcJSInfo source) = + object + ["source" .= source] instance FromJSON ResourceInfo where - parseJSON = withObject "ResourceInfo" $ \o -> do - version <- o .:? "version" .!= "" - url <- o .: "url" - contentLength <- o .:? "content-length" .!= (-1) - sha1 <- o .:? "sha1" .!= "" - sha256 <- o .:? "sha256" .!= "" - return ResourceInfo {..} + parseJSON = withObject "ResourceInfo" $ \o -> do + version <- o .:? "version" .!= "" + url <- o .: "url" + contentLength <- o .:? "content-length" .!= (-1) + sha1 <- o .:? "sha1" .!= "" + sha256 <- o .:? "sha256" .!= "" + return ResourceInfo {..} instance ToJSON ResourceInfo where - toJSON (ResourceInfo v u c s1 s256) = object $ - (if null v then [] else ["version" .= v]) ++ - ["url" .= u] ++ - (if c == -1 then [] else ["content-length" .= c]) ++ - (if null s1 then [] else ["sha1" .= s1]) ++ - (if null s256 then [] else ["sha256" .= s256]) + toJSON (ResourceInfo v u c s1 s256) = + object $ + (["version" .= v | not (null v)]) + ++ ["url" .= u] + ++ (["content-length" .= c | c /= (-1)]) + ++ (["sha1" .= s1 | not (null s1)]) + ++ (["sha256" .= s256 | not (null s256)]) instance FromJSON StackSetup where - parseJSON = withObject "StackSetup" $ \o -> do - stack <- o .: "stack" - sevenzexeInfo <- o .: "sevenzexe-info" - sevenzdllInfo <- o .: "sevenzdll-info" - portableGit <- o .: "portable-git" - msys2 <- o .: "msys2" - ghc <- o .: "ghc" - ghcjs <- o .: "ghcjs" - return StackSetup {..} + parseJSON = withObject "StackSetup" $ \o -> do + stack <- o .: "stack" + sevenzexeInfo <- o .: "sevenzexe-info" + sevenzdllInfo <- o .: "sevenzdll-info" + portableGit <- o .: "portable-git" + msys2 <- o .: "msys2" + ghc <- o .: "ghc" + ghcjs <- o .: "ghcjs" + return StackSetup {..} instance ToJSON StackSetup where - toJSON (StackSetup stack exe dll pgit msys2 ghc ghcjs) = object - ["stack" .= stack, - "sevenzexe-info" .= exe, - "sevenzdll-info" .= dll, - "portable-git" .= pgit, - "msys2" .= msys2, - "ghc" .= ghc, - "ghcjs" .= ghcjs] - + toJSON (StackSetup stack exe dll pgit msys2 ghc ghcjs) = + object + [ "stack" .= stack, + "sevenzexe-info" .= exe, + "sevenzdll-info" .= dll, + "portable-git" .= pgit, + "msys2" .= msys2, + "ghc" .= ghc, + "ghcjs" .= ghcjs + ] + +instance FromJSON GitHubReleases where + parseJSON = withObject "GitHubReleases" $ \o -> do + prerelease <- o .: "prerelease" + urls <- o .: "assets" >>= mapM (.: "browser_download_url") + return GitHubReleases {..} redirectToMirror :: Path -> ResourceInfo -> ResourceInfo redirectToMirror relPath (ResourceInfo ver url conLen s1 s256) = - let redirect = (++) ("https://mirrors.ustc.edu.cn/stackage/" ++ relPath ++ "/") . head . splitOn "?" . last . splitOn "/" in - ResourceInfo ver (redirect url) conLen s1 s256 - + let redirect = (++) ("https://mirrors.ustc.edu.cn/stackage/" ++ relPath ++ "/") . head . splitOn "?" . last . splitOn "/" + in ResourceInfo ver (redirect url) conLen s1 s256 -- download a file to given path -- sha-1 checksum is enabled when sha isn't empty string download :: URL -> FilePath -> (SHA1, SHA256) -> Bool -> IO () download url path sha force = do - let fileName = head (splitOn "?" (last (splitOn "/" url))) - let filePath = path fileName - putStrLn $ printf "Try to Download %s..." fileName - pathExists <- doesDirectoryExist path - unless pathExists (createDirectory path) - filePathExists <- doesFileExist filePath - if filePathExists && not force - then putStrLn $ printf "%s already exists. Just skip." filePath - else do - let args_ = [url, "--dir=" ++ path, - "--out=" ++ fileName ++ ".tmp", - "--file-allocation=none", "--quiet=true"] - - -- if sha1 isn't an empty string, append checksum option - let sha1Arg = (words (fst sha) >>= \s -> ["--checksum=sha-1=" ++ s]) - let sha256Arg = (words (snd sha) >>= \s -> ["--checksum=sha-256=" ++ s]) - let args = args_ ++ (if null sha256Arg then sha1Arg else sha256Arg) - - (exitCode, _, _) <- readProcessWithExitCode "aria2c" args "" - if exitCode == ExitSuccess - then callProcess "mv" [filePath ++ ".tmp", filePath] >> - putStrLn (printf "Downloaded %s to %s." fileName filePath) - else putStrLn $ printf "Download failure on %s" fileName + let fileName = head (splitOn "?" (last (splitOn "/" url))) + let filePath = path fileName + putStrLn $ printf "Try to Download %s..." fileName + pathExists <- doesDirectoryExist path + unless pathExists (createDirectory path) + filePathExists <- doesFileExist filePath + if filePathExists && not force + then putStrLn $ printf "%s already exists. Just skip." filePath + else do + let args_ = + [ url, + "--dir=" ++ path, + "--out=" ++ fileName ++ ".tmp", + "--file-allocation=none", + "--quiet=true" + ] + + -- if sha1 isn't an empty string, append checksum option + let sha1Arg = words (fst sha) >>= \s -> ["--checksum=sha-1=" ++ s] + let sha256Arg = words (snd sha) >>= \s -> ["--checksum=sha-256=" ++ s] + let args = args_ ++ (if null sha256Arg then sha1Arg else sha256Arg) + + (exitCode, _, _) <- readProcessWithExitCode "aria2c" args "" + if exitCode == ExitSuccess + then + callProcess "mv" [filePath ++ ".tmp", filePath] + >> putStrLn (printf "Downloaded %s to %s." fileName filePath) + else putStrLn $ printf "Download failure on %s" fileName updateChannels :: FilePath -> IO () -updateChannels basePath = - mapM_ loadChannel ["lts-haskell", "stackage-nightly", "stackage-snapshots", "stackage-content"] - where loadChannel channel = do - let destPath = basePath channel - exists <- doesDirectoryExist destPath - if exists - then do putStrLn $ printf "Start to pull latest %s channel" channel - callProcess "git" ["-C", destPath, "pull"] - putStrLn $ printf "Pull %s finish" channel - else do putStrLn $ printf "%s channel doesn't exist, start first clone" channel - callProcess "git" ["clone", "--depth", "1", - "https://github.com/commercialhaskell/" ++ channel ++ ".git", - destPath] - putStrLn $ printf "Clone %s finish" channel - +updateChannels basePath = + mapM_ loadChannel ["lts-haskell", "stackage-nightly", "stackage-snapshots", "stackage-content"] + where + loadChannel channel = do + let destPath = basePath channel + exists <- doesDirectoryExist destPath + if exists + then do + putStrLn $ printf "Start to pull latest %s channel" channel + callProcess "git" ["-C", destPath, "pull"] + putStrLn $ printf "Pull %s finish" channel + else do + putStrLn $ printf "%s channel doesn't exist, start first clone" channel + callProcess + "git" + [ "clone", + "--depth", + "1", + "https://github.com/commercialhaskell/" ++ channel ++ ".git", + destPath + ] + putStrLn $ printf "Clone %s finish" channel stackSetup :: FilePath -> FilePath -> IO () stackSetup bp setupPath = do - jr <- decodeFileEither setupPath :: IO (Either ParseException StackSetup) - r <- case jr of - Left err -> do - putStrLn (prettyPrintParseException err) - error "Parse setup yaml failure" - Right obj -> return obj + jr <- decodeFileEither setupPath :: IO (Either ParseException StackSetup) + r <- case jr of + Left err -> do + putStrLn (prettyPrintParseException err) + error "Parse setup yaml failure" + Right obj -> return obj - let (StackSetup stack exe dll pgit msys2 ghc ghcjs) = r + let (StackSetup stack exe dll pgit msys2 ghc ghcjs) = r - -- store stack - -- let filesToDownload = M.toList stack >>= M.toList . snd >>= return . snd + -- store stack + -- let filesToDownload = M.toList stack >>= M.toList . snd >>= return . snd - -- let dlEachStack (ResourceInfo _ u _ s1 s256) = download u (bp "stack") (s1, s256) False - -- in mapM_ dlEachStack filesToDownload + -- let dlEachStack (ResourceInfo _ u _ s1 s256) = download u (bp "stack") (s1, s256) False + -- in mapM_ dlEachStack filesToDownload - let newStack = M.map (M.map (redirectToMirror "stack")) stack + let newStack = M.map (M.map (redirectToMirror "stack")) stack - -- store 7z - let dl7z (ResourceInfo _ u _ s1 s256) = download u (bp "7z") (s1, s256) False - in do - dl7z exe - dl7z dll + -- store 7z + let dl7z (ResourceInfo _ u _ s1 s256) = download u (bp "7z") (s1, s256) False + in do + dl7z exe + dl7z dll - let newExe = redirectToMirror "7z" exe - let newDll = redirectToMirror "7z" dll + let newExe = redirectToMirror "7z" exe + let newDll = redirectToMirror "7z" dll - -- store portable git - let dlGit (ResourceInfo _ u _ s1 s256) = download u (bp "pgit") (s1, s256) False - in dlGit pgit + -- store portable git + let dlGit (ResourceInfo _ u _ s1 s256) = download u (bp "pgit") (s1, s256) False + in dlGit pgit - let newPgit = redirectToMirror "pgit" pgit + let newPgit = redirectToMirror "pgit" pgit + -- store ghc + let filesToDownload = map snd $ M.elems ghc >>= M.toList - -- store ghc - let filesToDownload = M.toList ghc >>= M.toList . snd >>= return . snd + let dlGhc (ResourceInfo _ u _ s1 s256) = download u (bp "ghc") (s1, s256) False + in mapM_ dlGhc filesToDownload - let dlGhc (ResourceInfo _ u _ s1 s256) = download u (bp "ghc") (s1, s256) False - in mapM_ dlGhc filesToDownload + let newGhc = M.map (M.map (redirectToMirror "ghc")) ghc - let newGhc = M.map (M.map (redirectToMirror "ghc")) ghc + -- store msys2 + let filesToDownload = snd <$> M.toList msys2 - -- store msys2 - let filesToDownload = snd <$> M.toList msys2 + let dlMsys2 (ResourceInfo _ u _ s1 s256) = download u (bp "msys2") (s1, s256) False + in mapM_ dlMsys2 filesToDownload - let dlMsys2 (ResourceInfo _ u _ s1 s256) = download u (bp "msys2") (s1, s256) False - in mapM_ dlMsys2 filesToDownload + let newMsys2 = M.map (redirectToMirror "msys2") msys2 - let newMsys2 = M.map (redirectToMirror "msys2") msys2 - - encodeFile (bp "stack-setup.yaml") (StackSetup newStack newExe newDll newPgit newMsys2 newGhc ghcjs) - putStrLn $ printf "Stack setup successfully processed" + encodeFile (bp "stack-setup.yaml") (StackSetup newStack newExe newDll newPgit newMsys2 newGhc ghcjs) + putStrLn $ printf "Stack setup successfully processed" syncStack :: FilePath -> IO () syncStack basePath = do - putStrLn "start to sync stack" - download "https://api.github.com/repos/commercialhaskell/stack/releases/latest" - "/tmp" - ("", "") - True - let filename = "/tmp/latest" - text <- BS.readFile filename - let (Object latestInfo) = case A.decode text of - Just o -> o :: Value - _ -> error "decode latest fail!" - let (Just (Bool prerelease)) = KM.lookup "prerelease" latestInfo - let (Just (Array assets)) = KM.lookup "assets" latestInfo - unless prerelease (syncAssets assets) - where syncAssets = mapM_ syncAsset - syncAsset assetValue = do - let (Object asset) = assetValue - let (Just (String url)) = KM.lookup "browser_download_url" asset - download (T.unpack url) - (basePath "stack") - ("", "") - False - - + putStrLn "start to sync stack" + download + "https://api.github.com/repos/commercialhaskell/stack/releases/latest" + "/tmp" + ("", "") + True + let filename = "/tmp/latest" + text <- BS.readFile filename + let latestInfo = case A.decode text of + Just o -> o :: GitHubReleases + _ -> error "decode latest fail!" + let isPreRelease = prerelease latestInfo + let assetsURLs = urls latestInfo + unless isPreRelease (syncAssets assetsURLs) + where + syncAssets = mapM_ syncAsset + syncAsset urlValue = do + download + urlValue + (basePath "stack") + ("", "") + False main :: IO () main = do - -- specify what place to save the mirror TO - basePath <- getEnv "TO" + -- specify what place to save the mirror TO + basePath <- getEnv "TO" - -- update channel - updateChannels basePath + -- update channel + updateChannels basePath - -- load snapshots - download "https://www.stackage.org/download/snapshots.json" basePath ("", "") True + -- load snapshots + download "https://www.stackage.org/download/snapshots.json" basePath ("", "") True - -- load stack setup - download "https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml" - "/tmp" - ("", "") - True + -- load stack setup + download + "https://raw.githubusercontent.com/fpco/stackage-content/master/stack/stack-setup-2.yaml" + "/tmp" + ("", "") + True - stackSetup basePath "/tmp/stack-setup-2.yaml" + stackSetup basePath "/tmp/stack-setup-2.yaml" - -- sync stack from github - syncStack basePath + -- sync stack from github + syncStack basePath - putStrLn "sync finish" + putStrLn "sync finish" diff --git a/yum-sync/Dockerfile b/yum-sync/Dockerfile index 949e2a4..d7b20de 100644 --- a/yum-sync/Dockerfile +++ b/yum-sync/Dockerfile @@ -1,7 +1,6 @@ FROM ustcmirror/base:debian LABEL maintainer="Keyu Tao " -RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' > /etc/apt/sources.list.d/backports.list && \ - apt update && apt install -y dnf createrepo-c dnf-plugins-core python3 python3-requests && \ +RUN apt update && apt install -y dnf createrepo-c dnf-plugins-core python3 python3-requests && \ sed 's/enabled = true/enabled = false/' -i /etc/dnf/plugins/local.conf ADD tunasync /usr/local/lib/tunasync ADD sync.sh /