Add code coverage using bisect_ppx (oxcaml#352)

Author: lukemaurer

.github/workflows/coverage.yml

@@ -0,0 +1,80 @@
+name: coverage-linux
+on: [push, pull_request]
+jobs:
+  build:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: flambda2_coverage
+            config: --enable-middle-end=flambda2 --enable-coverage
+            ocamlparam: ''
+
+    env:
+      J: "3"
+
+    steps:
+    - name: Checkout the Flambda backend repo
+      uses: actions/checkout@master
+      with:
+        path: 'flambda_backend'
+
+    # This adds a switch at $GITHUB_WORKSPACE/_opam
+    - name: Set up OCaml 4.12
+      uses: ocaml/setup-ocaml@v2
+      with:
+        ocaml-compiler: 4.12
+        opam-pin: false
+        opam-depext: false
+
+    - name: Get host build environment from cache
+      uses: actions/cache@v1
+      id: cache
+      with:
+        path: ${{ github.workspace }}/_opam
+        key: ${{ matrix.os }}-cache-ocaml-412-dune-2.9.1-g7606d586
+
+    - name: Build Dune
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        opam pin dune --yes \
+          git+https://github.com/ocaml-flambda/dune#special_dune
+
+    - name: Build bisect_ppx
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        # bisect_ppx fails to build with current ppxlib
+        opam pin ppxlib 0.23.0 --yes
+        opam pin bisect_ppx --yes \
+          git+https://github.com/lukemaurer/bisect_ppx#html-report-collate-fix
+
+    - name: Trim cached files
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        # Remove unneeded parts to shrink cache file
+        rm -rf _opam/{lib/ocaml/expunge,bin/*.byte}
+
+    - name: Configure Flambda backend
+      working-directory: flambda_backend
+      run: |
+        autoconf
+        ./configure \
+          --prefix=$GITHUB_WORKSPACE/_install \
+          --with-dune=$GITHUB_WORKSPACE/_opam/bin/dune \
+          ${{ matrix.config }}
+
+
+    - name: Build, install and test Flambda backend
+      working-directory: flambda_backend
+      run: opam exec -- make ci
+      env:
+        BUILD_OCAMLPARAM: ${{ matrix.ocamlparam }}
+
+    - name: Publish coverage report
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage
+        path: flambda_backend/_coverage/**

.gitignore

@@ -43,6 +43,7 @@ _compare/
 _runtest/
 autom4te.cache
 _install/
+_coverage/
 
 /Makefile.config
 /config.log

HACKING.md

@@ -9,6 +9,7 @@ want to modify the Flambda backend.  Jump to:
   - [Rebuilding during dev work](#rebuilding-during-dev-work)
   - [Running tests](#running-tests)
   - [Running only part of the upstream testsuite](#running-only-part-of-the-upstream-testsuite)
+  - [Running tests with coverage analysis](#running-tests-with-coverage-analysis)
   - [Running the compiler produced by "make hacking" on an example without the stdlib](#running-the-compiler-produced-by-make-hacking-on-an-example-without-the-stdlib)
   - [Getting the compilation command for a stdlib file](#getting-the-compilation-command-for-a-stdlib-file)
   - [Bootstrapping the ocaml subtree](#bootstrapping-the-ocaml-subtree)
@@ -134,6 +135,27 @@ where `<FLAMBDA_BACKEND>` is the path to your clone.
 You may also need the `CAML_LD_LIBRARY_PATH` setting depending on what you are testing (see `Makefile.in` at the
 root).
 
+## Running tests with coverage analysis
+
+Coverage analysis is available for the Flambda backend tests (that is, just the
+ones run by `make runtest`), which are intended to provide good coverage on
+their own. We use `bisect_ppx` to perform the analysis. Since binaries
+instrumented with `bisect_ppx` have coverage enabled unconditionally, coverage
+support is disabled by default at compile time.
+
+Coverage requires the `bisect_ppx` package to be installed in your OPAM switch.
+Since no OPAM environment is available when building the final compiler, we
+instead enable coverage on the boot compiler and run the tests directly on the
+boot compiler.
+
+To enable coverage, pass the `--enable-coverage` flag to `./configure`.
+(Remember to clean, as by `git clean -dfX`, whenever re-running
+`./configure`). When coverage is enabled, `make boot-runtest` will
+run the tests on the boot compiler and produce coverage data, and
+`make coverage` will produce an HTML report in `_coverage/`. Alternatively,
+with coverage enabled, `make ci` will build the boot compiler, run the
+tests, and produce the report.
+
 ## Running the compiler produced by "make hacking" on an example without the stdlib
 
 For small examples that don't need the stdlib or any other library provided by the

Makefile

@@ -7,6 +7,14 @@ ws_boot   = --root=. --workspace=duneconf/boot.ws
 ws_runstd = --root=. --workspace=duneconf/runtime_stdlib.ws
 ws_main   = --root=. --workspace=duneconf/main.ws
 
+ifeq ($(coverage),yes)
+  coverage_dune_flags=--instrument-with bisect_ppx
+  ocaml_subdirs_to_ignore=otherlibs
+else
+  coverage_dune_flags=
+  ocaml_subdirs_to_ignore=
+endif
+
 define dune_boot_context
 (lang dune 2.8)
 ; We need to call the boot context "default" so that dune selects it for merlin
@@ -42,15 +50,18 @@ endef
 
 
 .DEFAULT_GOAL := compiler
-.PHONY: boot-compiler runtime-stdlib compiler runtest
+.PHONY: boot-compiler boot-runtest runtime-stdlib compiler runtest
 
 boot-compiler: _build/_bootinstall
-	$(dune) build $(ws_boot) \
+	$(dune) build $(ws_boot) $(coverage_dune_flags) \
 	  boot_ocamlopt.exe \
 	  ocaml/main_native.exe \
 	  ocaml/tools/ocamlmklib_native.exe \
 	  ocaml/tools/ocamldep_native.exe
 
+boot-runtest: boot-compiler
+	$(dune) runtest $(ws_boot) $(coverage_dune_flags) --force
+
 runtime-stdlib: boot-compiler
 	$(dune) build $(ws_runstd) --only-package=ocaml_runtime_stdlib @install
 #	dune does not believe the compiler can make .cmxs unless the following file exists
@@ -71,7 +82,19 @@ duneconf/main.ws: export contents = $(dune_main_context)
 duneconf/%.ws:
 	echo "$$contents" > $@
 
-_build/_bootinstall: ocaml/Makefile.config duneconf/boot.ws duneconf/runtime_stdlib.ws duneconf/main.ws
+# We need to disable ocaml/otherlibs when compiling with coverage, because we
+# need to compile against the user's opam instead. Unfortunately, Dune gives us
+# no nicer way of declaring data_only_dirs differently in different workspaces,
+# so we have to output a file to be included in ocaml/dune.
+#
+# Also, Dune only allows one (data_only_dirs) declaration per file, so here we
+# have to account for the declaration that would already have been in
+# ocaml/dune.
+ocaml/dirs-to-ignore.inc:
+	echo "(data_only_dirs yacc $(ocaml_subdirs_to_ignore))" > $@
+
+_build/_bootinstall: ocaml/Makefile.config duneconf/boot.ws duneconf/runtime_stdlib.ws duneconf/main.ws \
+	ocaml/dirs-to-ignore.inc
 
 # natdynlinkops2:
 # We need to augment dune's substitutions so this part isn't so
@@ -267,7 +290,17 @@ runtest-upstream: _install
              fi)
 
 .PHONY: ci
-ci: runtest runtest-upstream
+ifeq ($(coverage),yes)
+ci: ci-coverage
+else
+ci: ci-no-coverage
+endif
+
+.PHONY: ci-no-coverage
+ci-no-coverage: runtest runtest-upstream
+
+.PHONY: ci-coverage
+ci-coverage: boot-runtest coverage
 
 # This target is like a polling version of upstream "make ocamlopt"
 .PHONY: hacking
@@ -389,3 +422,12 @@ build_upstream: ocaml/config.status
 install_upstream: build_upstream
 	(cd _build_upstream && $(MAKE) install)
 	cp ocaml/VERSION $(prefix)/lib/ocaml/
+
+.PHONY: coverage
+coverage: boot-runtest
+	rm -rf _coverage
+	bisect-ppx-report html --tree -o _coverage \
+	  --coverage-path=_build/default \
+		--source-path=. \
+	  --source-path=_build/default
+	@echo Coverage report generated in _coverage/index.html

Makefile.config.in

@@ -1,3 +1,4 @@
 install_prefix = @prefix@
 dune = @dune@
 middle_end = @middle_end@
+coverage = @coverage@

configure.ac

@@ -36,9 +36,18 @@ AC_ARG_ENABLE([middle-end],
     [*], [AC_MSG_ERROR([Bad middle end (not closure, flambda or flambda2)])])],
   [AC_MSG_ERROR([--enable-middle-end=closure|flambda|flambda2 must be provided])])
 
+AC_ARG_ENABLE([coverage],
+  [AS_HELP_STRING([--enable-coverage],
+    [Run compiler tests instrumented to output coverage data using bisect_ppx
+     (WARNING: Cannot build an installable compiler with this flag enabled.
+     Mainly intended for use in CI.)])],
+  [coverage=yes],
+  [coverage=no])
+
 AC_SUBST([prefix])
 AC_SUBST([middle_end])
 AC_SUBST([dune])
+AC_SUBST([coverage])
 
 # Don't error on options that this configure script doesn't understand but
 # the ocaml/ one does.

dune

@@ -66,6 +66,7 @@
   (:standard -principal -w -9))
  (ocamlopt_flags
   (:include %{project_root}/ocamlopt_flags.sexp))
+ (instrumentation (backend bisect_ppx))
  (modules_without_implementation
   backend_intf
   cmx_format
@@ -282,12 +283,14 @@
    ))
  (ocamlopt_flags
   (:include %{project_root}/ocamlopt_flags.sexp))
+ (instrumentation (backend bisect_ppx))
  (libraries ocamlcommon dwarf_flags asm_targets)
  (modules flambda_backend_args flambda_backend_flags))
 
 (executable
  (name flambda_backend_main)
  (modes byte)
+ (instrumentation (backend bisect_ppx))
  (flags (:standard -principal))
  (libraries
   flambda_backend_driver
@@ -307,6 +310,7 @@
 (executable
  (name flambda_backend_main_native)
  (modes native)
+ (instrumentation (backend bisect_ppx))
  (flags
   (:standard -principal))
  (libraries
@@ -328,6 +332,7 @@
 (executable
  (name boot_ocamlopt)
  (modes native)
+ (instrumentation (backend bisect_ppx))
  (flags
   (:standard -principal))
  (libraries

middle_end/flambda2/algorithms/dune

@@ -3,6 +3,7 @@
 (library
  (name flambda2_algorithms)
  (wrapped true)
+ (instrumentation (backend bisect_ppx))
  (flags
   (:standard -principal))
  (ocamlopt_flags

middle_end/flambda2/bound_identifiers/dune

@@ -3,6 +3,7 @@
 (library
  (name flambda2_bound_identifiers)
  (wrapped true)
+ (instrumentation (backend bisect_ppx))
  (flags
   (:standard
    -principal

middle_end/flambda2/cmx/dune

@@ -3,6 +3,7 @@
 (library
  (name flambda2_cmx)
  (wrapped true)
+ (instrumentation (backend bisect_ppx))
  (flags
   (:standard
    -principal