Refactor AppVeyor scripts to enable all ports (#10065)

* Refactor AppVeyor script for testing all ports * Update to FlexDLL 0.39 * Allow forcing Cygwin upgrade * Split off the old-compilers job * Display base addresses of DLLs * Update HACKING.adoc
gretay-js · Dec 16, 2020 · 686a7a3 · 686a7a3
1 parent d07d6c3
commit 686a7a3
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 68 deletions.
diff --git a/HACKING.adoc b/HACKING.adoc
@@ -389,6 +389,29 @@ entry needed)" in one of your commit messages -- but in general all
 patches submitted should come with a Changes entry; see the guidelines
 in link:CONTRIBUTING.md[].
 
+The Windows ports take a long time to test - INRIA's precheck service is the
+best to use when all 6 Windows ports need testing for a branch, but the
+AppVeyor scripts also support the other ports. The matrix is controlled by
+the following environment variables, which should be set in appveyor.yml:
+
+- `PORT` - this must be set on each job. Either `mingw`, `msvc` or `cygwin`
+  followed by `32` or `64`.
+- `BOOTSTRAP_FLEXDLL` - must be set on each job. Either `true` or `false`.
+  At present, must be `false` for Cygwin builds. Controls whether flexlink
+  is bootstrapped as part of the test or installed from a binary archive.
+- `FORCE_CYGWIN_UPGRADE`. Default: `0`. Set to `1` to force an upgrade of
+  Cygwin packages as part of the build. Normally a full upgrade is only
+  triggered if the packages installed require it.
+- `BUILD_MODE`. Default: `world.opt`. Either `world.opt`, `steps`, or `C`.
+  Controls whether the build uses the `world.opt` target or the classic
+  `world`, `opt`, `opt.opt` targets. The `C` build is a fast test used to
+  build just enough of the tree to cover the C sources (it's used to test
+  old MSVC compilers).
+- `SDK`. Defaults to Visual Studio 2015. Specifies the exact command to run
+  to set-up the Microsoft build environment.
+- `CYGWIN_DIST`. Default: `64`. Either `64` or `32`, selects 32-bit or 64-bit
+  Cygwin as the build environment.
+
 ==== INRIA's Continuous Integration (CI)
 
 INRIA provides a Jenkins continuous integration service that OCaml

diff --git a/appveyor.yml b/appveyor.yml
@@ -26,11 +26,21 @@ environment:
     CYG_ROOT: C:/cygwin64
     CYG_MIRROR: http://mirrors.kernel.org/sourceware/cygwin/
     CYG_CACHE: C:/cygwin64/var/cache/setup
-    FLEXDLL_VERSION: 0.38
+    FLEXDLL_VERSION: 0.39
     OCAMLRUNPARAM: v=0,b
+    FORCE_CYGWIN_UPGRADE: 0
+    BUILD_MODE: world.opt
   matrix:
     - PORT: mingw32
+      BOOTSTRAP_FLEXDLL: true
     - PORT: msvc64
+      BOOTSTRAP_FLEXDLL: false
+      BUILD_MODE: steps
+    - PORT: msvc32
+      BOOTSTRAP_FLEXDLL: false
+      BUILD_MODE: C
+      SDK: |-
+        "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86
 
 matrix:
   fast_finish: true

diff --git a/tools/ci/appveyor/appveyor_build.cmd b/tools/ci/appveyor/appveyor_build.cmd
@@ -58,7 +58,7 @@ goto :EOF
 
 :UpgradeCygwin
 if "%CYGWIN_INSTALL_PACKAGES%" neq "" "%CYG_ROOT%\setup-x86_64.exe" --quiet-mode --no-shortcuts --no-startmenu --no-desktop --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages %CYGWIN_INSTALL_PACKAGES:~1% > nul
-for %%P in (%CYGWIN_COMMANDS%) do "%CYG_ROOT%\bin\%%P.exe" --version > nul || set CYGWIN_UPGRADE_REQUIRED=1
+for %%P in (%CYGWIN_COMMANDS%) do "%CYG_ROOT%\bin\%%P.exe" --version 2> nul > nul || set CYGWIN_UPGRADE_REQUIRED=1
 "%CYG_ROOT%\bin\bash.exe" -lc "cygcheck -dc %CYGWIN_PACKAGES%"
 if %CYGWIN_UPGRADE_REQUIRED% equ 1 (
   echo Cygwin package upgrade required - please go and drink coffee
@@ -72,12 +72,9 @@ chcp 65001 > nul
 rem This must be kept in sync with appveyor_build.sh
 set BUILD_PREFIX=🐫реализация
 git worktree add "..\%BUILD_PREFIX%-%PORT%" -b appveyor-build-%PORT%
-if "%PORT%" equ "msvc64" (
-  git worktree add "..\%BUILD_PREFIX%-msvc32" -b appveyor-build-%PORT%32
-)
 
 cd "..\%BUILD_PREFIX%-%PORT%"
-if "%PORT%" equ "mingw32" (
+if "%BOOTSTRAP_FLEXDLL%" equ "true" (
   git submodule update --init flexdll
 )
 
@@ -104,9 +101,21 @@ if "%PORT%" equ "mingw32" (
   set CYGWIN_PACKAGES=%CYGWIN_PACKAGES% mingw64-i686-gcc-core mingw64-i686-runtime
   set CYGWIN_COMMANDS=%CYGWIN_COMMANDS% i686-w64-mingw32-gcc cygcheck
 )
+if "%PORT%" equ "mingw64" (
+  set CYGWIN_PACKAGES=%CYGWIN_PACKAGES% mingw64-x86_64-gcc-core
+  set CYGWIN_COMMANDS=%CYGWIN_COMMANDS% x86_64-w64-mingw32-gcc
+)
+if "%PORT%" equ "cygwin32" (
+  set CYGWIN_PACKAGES=%CYGWIN_PACKAGES% cygwin32-gcc-core flexdll
+  set CYGWIN_COMMANDS=%CYGWIN_COMMANDS% i686-pc-cygwin-gcc flexlink
+)
+if "%PORT%" equ "cygwin64" (
+  set CYGWIN_PACKAGES=%CYGWIN_PACKAGES% gcc-core flexdll
+  set CYGWIN_COMMANDS=%CYGWIN_COMMANDS% x86_64-pc-cygwin-gcc flexlink
+)
 
 set CYGWIN_INSTALL_PACKAGES=
-set CYGWIN_UPGRADE_REQUIRED=0
+set CYGWIN_UPGRADE_REQUIRED=%FORCE_CYGWIN_UPGRADE%
 
 for %%P in (%CYGWIN_PACKAGES%) do call :CheckPackage %%P
 call :UpgradeCygwin
@@ -116,23 +125,24 @@ call :UpgradeCygwin
 goto :EOF
 
 :build
-if "%PORT%" equ "msvc64" (
-  setlocal
-  call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat"
+rem Testing %SDK% is tricky, since it can contain double-quotes. The "trick",
+rem is to make SDK_TEST the second character of %SDK%. If %SDK% is un-set then
+rem SDK_TEST will be the literal string %SDK:~1,1%, obviously. However, that
+rem means %SDK_TEST:~1,1% only expands to the empty string if SDK was itself
+rem un-set. <sigh>
+set SDK_TEST=%SDK:~1,1%
+if "%SDK_TEST:~1,1%" neq "" (
+  if "%PORT%" equ "msvc64" set SDK=call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat"
+  if "%PORT%" equ "msvc32" set SDK=call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\vcvars32.bat"
+) else (
+  set SDK=call %SDK%
 )
-rem Do the main build (either msvc64 or mingw32)
-"%CYG_ROOT%\bin\bash.exe" -lc "$APPVEYOR_BUILD_FOLDER/tools/ci/appveyor/appveyor_build.sh" || exit /b 1
 
-if "%PORT%" neq "msvc64" goto :EOF
+%SDK%
 
-rem Reconfigure the environment and run the msvc32 partial build
-endlocal
-call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x86
-"%CYG_ROOT%\bin\bash.exe" -lc "$APPVEYOR_BUILD_FOLDER/tools/ci/appveyor/appveyor_build.sh msvc32-only" || exit /b 1
+"%CYG_ROOT%\bin\bash.exe" -lc "$APPVEYOR_BUILD_FOLDER/tools/ci/appveyor/appveyor_build.sh" || exit /b 1
 goto :EOF
 
 :test
-rem Reconfigure the environment for the msvc64 build
-call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat"
-"%CYG_ROOT%\bin\bash.exe" -lc "$APPVEYOR_BUILD_FOLDER/tools/ci/appveyor/appveyor_build.sh test" || exit /b 1
+if "%BUILD_MODE%" neq "C" "%CYG_ROOT%\bin\bash.exe" -lc "$APPVEYOR_BUILD_FOLDER/tools/ci/appveyor/appveyor_build.sh test" || exit /b 1
 goto :EOF
diff --git a/tools/ci/appveyor/appveyor_build.sh b/tools/ci/appveyor/appveyor_build.sh
@@ -51,12 +51,20 @@ function run {
 # $2: the prefix to use to install
 function set_configuration {
     case "$1" in
-        mingw)
+        cygwin*)
+            dep='--disable-dependency-generation'
+        ;;
+        mingw32)
             build='--build=i686-pc-cygwin'
             host='--host=i686-w64-mingw32'
             dep='--disable-dependency-generation'
         ;;
-        msvc)
+        mingw64)
+            build='--build=i686-pc-cygwin'
+            host='--host=x86_64-w64-mingw32'
+            dep='--disable-dependency-generation'
+        ;;
+        msvc32)
             build='--build=i686-pc-cygwin'
             host='--host=i686-pc-windows'
             dep='--disable-dependency-generation'
@@ -88,33 +96,29 @@ OCAMLROOT=$(echo "$PROGRAMFILES/Бактріан🐫" | cygpath -f - -m)
 # This must be kept in sync with appveyor_build.cmd
 BUILD_PREFIX=🐫реализация
 
-PATH=$(echo "$OCAMLROOT" | cygpath -f -)/bin/flexdll:$PATH
+if [[ $BOOTSTRAP_FLEXDLL = 'false' ]] ; then
+  case "$PORT" in
+    cygwin*) ;;
+    *) PATH=$(echo "$OCAMLROOT" | cygpath -f -)/bin/flexdll:$PATH;;
+  esac
+fi
 
 case "$1" in
   install)
-    mkdir -p "$OCAMLROOT/bin/flexdll"
-    cd "$APPVEYOR_BUILD_FOLDER/../flexdll"
-    # msvc64 objects need to be compiled with VS2015, so are copied later from
-    # a source build.
-    for f in flexdll.h flexlink.exe flexdll*_msvc.obj default*.manifest ; do
-      cp "$f" "$OCAMLROOT/bin/flexdll/"
-    done
-    if [[ $PORT = 'msvc64' ]] ; then
-      echo 'eval $($APPVEYOR_BUILD_FOLDER/tools/msvs-promote-path)' \
-        >> ~/.bash_profile
+    if [[ $BOOTSTRAP_FLEXDLL = 'false' ]] ; then
+      mkdir -p "$OCAMLROOT/bin/flexdll"
+      cd "$APPVEYOR_BUILD_FOLDER/../flexdll"
+      # The objects are always built from the sources
+      for f in flexdll.h flexlink.exe default*.manifest ; do
+        cp "$f" "$OCAMLROOT/bin/flexdll/"
+      done
     fi
-    ;;
-  msvc32-only)
-    cd "$APPVEYOR_BUILD_FOLDER/../$BUILD_PREFIX-msvc32"
-
-    set_configuration msvc "$OCAMLROOT-msvc32"
-
-    run "$MAKE world" $MAKE world
-    run "$MAKE runtimeopt" $MAKE runtimeopt
-    run "$MAKE -C otherlibs/systhreads libthreadsnat.lib" \
-         $MAKE -C otherlibs/systhreads libthreadsnat.lib
-
-    exit 0
+    case "$PORT" in
+      msvc*)
+        echo 'eval $($APPVEYOR_BUILD_FOLDER/tools/msvs-promote-path)' \
+          >> ~/.bash_profile
+        ;;
+    esac
     ;;
   test)
     FULL_BUILD_PREFIX="$APPVEYOR_BUILD_FOLDER/../$BUILD_PREFIX"
@@ -124,6 +128,11 @@ case "$1" in
           "$FULL_BUILD_PREFIX-$PORT/tools/check-symbol-names" \
           $FULL_BUILD_PREFIX-$PORT/runtime/*.a
     fi
+    if [[ $PORT = 'mingw64' ]] ; then
+      export PATH="$PATH:/usr/x86_64-w64-mingw32/sys-root/mingw/bin"
+    elif [[ $PORT = 'mingw32' ]] ; then
+      export PATH="$PATH:/usr/i686-w64-mingw32/sys-root/mingw/bin"
+    fi
     run "test $PORT" $MAKE -C "$FULL_BUILD_PREFIX-$PORT" tests
     run "install $PORT" $MAKE -C "$FULL_BUILD_PREFIX-$PORT" install
     if [[ $PORT = 'msvc64' ]] ; then
@@ -152,42 +161,59 @@ case "$1" in
     if [[ $PORT = 'msvc64' ]] ; then
       # Ensure that make distclean can be run from an empty tree
       run "$MAKE distclean" $MAKE distclean
+    fi
+
+    if [[ $BOOTSTRAP_FLEXDLL = 'false' ]] ; then
       tar -xzf "$APPVEYOR_BUILD_FOLDER/flexdll.tar.gz"
       cd "flexdll-$FLEXDLL_VERSION"
-      $MAKE MSVC_DETECT=0 CHAINS=msvc64 support
-      cp flexdll*_msvc64.obj "$OCAMLROOT/bin/flexdll/"
+      $MAKE MSVC_DETECT=0 CHAINS=${PORT%32} support
+      cp -f *.obj "$OCAMLROOT/bin/flexdll/" || \
+      cp -f *.o "$OCAMLROOT/bin/flexdll/"
       cd ..
     fi
 
-    if [[ $PORT = 'msvc64' ]] ; then
-      set_configuration msvc64 "$OCAMLROOT"
-    else
-      set_configuration mingw "$OCAMLROOT-mingw32"
-    fi
-
-    cd "$APPVEYOR_BUILD_FOLDER/../$BUILD_PREFIX-$PORT"
+    set_configuration "$PORT" "$OCAMLROOT"
 
     export TERM=ansi
 
-    if [[ $PORT = 'mingw32' ]] ; then
-      set -o pipefail
-      # For an explanation of the sed command, see
-      # https://github.com/appveyor/ci/issues/1824
-      script --quiet --return --command \
-        "$MAKE -C ../$BUILD_PREFIX-mingw32 flexdll && "\
-"$MAKE -C ../$BUILD_PREFIX-mingw32 world.opt" \
-        "../$BUILD_PREFIX-mingw32/build.log" |
-          sed -e 's/\d027\[K//g' \
-              -e 's/\d027\[m/\d027[0m/g' \
-              -e 's/\d027\[01\([m;]\)/\d027[1\1/g'
-    else
+    case "$BUILD_MODE" in
+      world.opt)
+        set -o pipefail
+        # For an explanation of the sed command, see
+        # https://github.com/appveyor/ci/issues/1824
+        script --quiet --return --command \
+          "( test "$BOOTSTRAP_FLEXDLL" = 'false' || "\
+"$MAKE -C ../$BUILD_PREFIX-$PORT flexdll ) && "\
+"$MAKE -C ../$BUILD_PREFIX-$PORT world.opt" \
+          "../$BUILD_PREFIX-$PORT/build.log" |
+            sed -e 's/\d027\[K//g' \
+                -e 's/\d027\[m/\d027[0m/g' \
+                -e 's/\d027\[01\([m;]\)/\d027[1\1/g';;
+    steps)
       run "C deps: runtime" make -j64 -C runtime setup-depend
       run "C deps: win32unix" make -j64 -C otherlibs/win32unix setup-depend
       run "$MAKE world" $MAKE world
       run "$MAKE bootstrap" $MAKE bootstrap
       run "$MAKE opt" $MAKE opt
-      run "$MAKE opt.opt" $MAKE opt.opt
-    fi
+      run "$MAKE opt.opt" $MAKE opt.opt;;
+    C)
+      run "$MAKE world" $MAKE world
+      run "$MAKE runtimeopt" $MAKE runtimeopt
+      run "$MAKE -C otherlibs/systhreads libthreadsnat.lib" \
+           $MAKE -C otherlibs/systhreads libthreadsnat.lib;;
+    *)
+      echo "Unrecognised build: $BUILD_MODE"
+      exit 1
+    esac
+
+    echo DLL base addresses
+    case "$PORT" in
+      *32)
+        ARG='-4';;
+      *64)
+        ARG='-8';;
+    esac
+    find "../$BUILD_PREFIX-$PORT" -type f -name \*.dll | xargs rebase -i "$ARG"
 
     ;;
 esac