Control the flexdll bootstrap with configure

On Windows and Cygwin, if the submodule is initialised, configure will
pick it up (this can be disabled by passing --without-flexdll to
configure). --with-flexdll can also be used to specify a different
location for the FlexDLL sources (e.g. when building using opam) and
sources specified this way are first copied to flexdll-sources/

.gitattributes

@@ -151,6 +151,7 @@ menhir-bench.bash typo.missing-header typo.utf8
 
 /tools/ci/appveyor/appveyor_build.cmd text eol=crlf
 
+aclocal.m4 typo.tab
 configure.ac text eol=lf
 build-aux/compile text eol=lf
 build-aux/config.guess text eol=lf

.gitignore

@@ -84,6 +84,8 @@ _build
 /emacs/ocamltags
 /emacs/*.elc
 
+/flexdll-sources
+
 /lambda/runtimedef.ml
 
 /lex/parser.ml

Changes

@@ -358,12 +358,13 @@ Working version
   (Gabriel Scherer, review by Sébastien Hinderer and David Allsopp,
    report by Ralph Seichter)
 
-- #10135: Overhaul the FlexDLL bootstrap process. A separate non-shared version
-  of the runtime is built, and shared artefacts are no longer built twice. The
-  flexdll target no longer has to be separately built, but is folded into the
-  default make target. flexlink.opt is now built automatically with opt.opt. If
-  the flexdll code is present, any flexlink in PATH is ignored. The Cygwin port
-  now also supports bootstrapping FlexDLL.
+- #10135: Overhaul the FlexDLL bootstrap process. It's now fully integrated
+  with the default build target and controlled by --with-flexdll which allows
+  externally downloaded sources for FlexDLL to be specified. A separate
+  non-shared version of the runtime is built, and shared artefacts are no longer
+  built twice. When bootstrapping, any flexlink in PATH is now ignored and the
+  Cygwin port also supports bootstrapping FlexDLL. flexlink.opt is automatically
+  built and installed as part of opt.opt/world.opt.
   (David Allsopp, review by Sébastien Hinderer)
 
 - #10156: configure script: fix sockets feature detection.

Makefile

@@ -88,10 +88,12 @@ else
 EXTRAPATH = PATH="otherlibs/win32unix:$(PATH)"
 endif
 
-BOOT_FLEXLINK_CMD=
 
-FLEXDLL_SUBMODULE_PRESENT := $(wildcard flexdll/Makefile)
-ifneq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
+  COLDSTART_DEPS =
+  BOOT_FLEXLINK_CMD =
+else
+  COLDSTART_DEPS = boot/ocamlruns$(EXE)
   BOOT_FLEXLINK_CMD = \
     FLEXLINK_CMD="../boot/ocamlruns$(EXE) ../boot/flexlink.byte$(EXE)"
 endif
@@ -154,25 +156,24 @@ boot/ocamlruns$(EXE):
 # runtime/ocamlrun is then installed to boot/ocamlrun and the stdlib artefacts
 # are copied to boot/
 .PHONY: coldstart
-ifeq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
-coldstart:
+coldstart: $(COLDSTART_DEPS)
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
 	$(MAKE) -C runtime all
 	$(MAKE) -C stdlib \
 	  OCAMLRUN='$$(ROOTDIR)/runtime/ocamlrun$(EXE)' \
 	  CAMLC='$$(BOOT_OCAMLC) $(USE_RUNTIME_PRIMS)' all
 else
-coldstart: boot/ocamlruns$(EXE)
 	$(MAKE) -C stdlib OCAMLRUN='$$(ROOTDIR)/boot/ocamlruns$(EXE)' \
     CAMLC='$$(BOOT_OCAMLC)' all
-	$(MAKE) -C flexdll $(FLEXLINK_BUILD_ENV) \
+	$(MAKE) -C $(FLEXDLL_SOURCES) $(FLEXLINK_BUILD_ENV) \
 	  OCAMLRUN='$$(ROOTDIR)/boot/ocamlruns$(EXE)' NATDYNLINK=false \
 	  OCAMLOPT='$(value BOOT_OCAMLC) $(USE_RUNTIME_PRIMS) $(USE_STDLIB)' \
 	  flexlink.exe
-	mv flexdll/flexlink.exe boot/flexlink.byte$(EXE)
-	$(MAKE) -C flexdll $(FLEXLINK_BUILD_ENV) support
-	mv flexdll/flexdll_*.$(O) boot/
+	mv $(FLEXDLL_SOURCES)/flexlink.exe boot/flexlink.byte$(EXE)
+	$(MAKE) -C $(FLEXDLL_SOURCES) $(FLEXLINK_BUILD_ENV) support
+	mv $(FLEXDLL_SOURCES)/flexdll_*.$(O) boot/
 	$(MAKE) -C runtime $(BOOT_FLEXLINK_CMD) all
-endif # ifeq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+endif # ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
 	cp runtime/ocamlrun$(EXE) boot/ocamlrun$(EXE)
 	cd boot; rm -f $(LIBFILES)
 	cd stdlib; cp $(LIBFILES) ../boot
@@ -252,7 +253,7 @@ opt.opt: checknative
 	$(MAKE) otherlibrariesopt
 	$(MAKE) ocamllex.opt ocamltoolsopt ocamltoolsopt.opt $(OCAMLDOC_OPT) \
 	  $(OCAMLTEST_OPT)
-ifneq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "true"
 	$(MAKE) flexlink.opt$(EXE)
 endif
 ifeq "$(WITH_OCAMLDOC)-$(STDLIB_MANPAGES)" "ocamldoc-true"
@@ -313,41 +314,52 @@ world.opt: checknative
 # Different git mechanism displayed depending on whether this source tree came
 # from a git clone or a source tarball.
 
-flexdll/Makefile:
-	@echo In order to bootstrap FlexDLL, you need to place the sources in
-	@echo flexdll.
+.PHONY: flexdll flexlink flexlink.opt
+
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
+flexdll flexlink flexlink.opt:
+	@echo It is no longer necessary to bootstrap FlexDLL with a separate
+	@echo make invocation. Simply place the sources for FlexDLL in a
+	@echo sub-directory.
 	@echo This can either be done by downloading a source tarball from
-	@echo \  http://alain.frisch.fr/flexdll.html
+	@echo \  https://github.com/alainfrisch/flexdll/releases
 	@if [ -d .git ]; then \
 	  echo or by checking out the flexdll submodule with; \
 	  echo \  git submodule update --init; \
 	else \
 	  echo or by cloning the git repository; \
 	  echo \  git clone https://github.com/alainfrisch/flexdll.git; \
 	fi
+	@echo "Then pass --with-flexdll=<dir> to configure and build as normal."
 	@false
 
+else
+
 .PHONY: flexdll
 flexdll: flexdll/Makefile
 	@echo WARNING! make flexdll is no longer required
 	@echo This target will be removed in a future release.
 
 .PHONY: flexlink
-flexlink: flexdll/Makefile
-	@echo WARNING! make flexlink is no longer required
-	@echo This target will be removed in a future release.
+flexlink:
+	@echo Bootstrapping just flexlink.exe is no longer supported
+	@echo Bootstrapping FlexDLL is now enabled with
+	@echo ./configure --with-flexdll
+	@false
 
-flexlink.opt$(EXE): flexdll/Makefile
-	$(MAKE) -C flexdll $(FLEXLINK_BUILD_ENV) \
+flexlink.opt$(EXE):
+	$(MAKE) -C $(FLEXDLL_SOURCES) $(FLEXLINK_BUILD_ENV) \
     OCAML_FLEXLINK='$(value OCAMLRUN) $$(ROOTDIR)/boot/flexlink.byte$(EXE)' \
 	  OCAMLOPT="../ocamlopt.opt$(EXE) -nostdlib -I ../stdlib" flexlink.exe
-	mv flexdll/flexlink.exe $@
+	mv $(FLEXDLL_SOURCES)/flexlink.exe $@
 
 partialclean::
 	rm -f flexlink.opt$(EXE)
+endif # ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
 
-INSTALL_COMPLIBDIR=$(DESTDIR)$(COMPLIBDIR)
-INSTALL_FLEXDLLDIR=$(INSTALL_LIBDIR)/flexdll
+INSTALL_COMPLIBDIR = $(DESTDIR)$(COMPLIBDIR)
+INSTALL_FLEXDLLDIR = $(INSTALL_LIBDIR)/flexdll
+FLEXDLL_MANIFEST = default$(filter-out _i386,_$(ARCH)).manifest
 
 # Installation
 .PHONY: install
@@ -428,18 +440,18 @@ endif
 	if test -n "$(WITH_DEBUGGER)"; then \
 	  $(MAKE) -C debugger install; \
 	fi
-ifneq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "true"
 ifeq "$(TOOLCHAIN)" "msvc"
-	$(INSTALL_DATA) flexdll/default$(filter-out _i386,_$(ARCH)).manifest \
+	$(INSTALL_DATA) $(FLEXDLL_SOURCES)/$(FLEXDLL_MANIFEST) \
     "$(INSTALL_BINDIR)/"
 endif
-	$(MKDIR) "$(INSTALL_FLEXDLLDIR)"
-	$(INSTALL_DATA) boot/flexdll_*.$(O) "$(INSTALL_FLEXDLLDIR)"
 ifeq "$(INSTALL_BYTECODE_PROGRAMS)" "true"
 	$(INSTALL_PROG) \
 	  boot/flexlink.byte$(EXE) "$(INSTALL_BINDIR)/flexlink.byte$(EXE)"
 endif # ifeq "$(INSTALL_BYTECODE_PROGRAMS)" "true"
-endif # ifneq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+	$(MKDIR) "$(INSTALL_FLEXDLLDIR)"
+	$(INSTALL_DATA) boot/flexdll_*.$(O) "$(INSTALL_FLEXDLLDIR)"
+endif # ifeq "$(BOOTSTRAPPING_FLEXDLL)" "true"
 	$(INSTALL_DATA) Makefile.config "$(INSTALL_LIBDIR)"
 ifeq "$(INSTALL_BYTECODE_PROGRAMS)" "true"
 	if test -f ocamlopt$(EXE); then $(MAKE) installopt; else \
@@ -531,7 +543,7 @@ installoptopt:
 	   $(LN) ocamlc.opt$(EXE) ocamlc$(EXE); \
 	   $(LN) ocamlopt.opt$(EXE) ocamlopt$(EXE); \
 	   $(LN) ocamllex.opt$(EXE) ocamllex$(EXE)
-ifneq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "true"
 	$(INSTALL_PROG) flexlink.opt$(EXE) "$(INSTALL_BINDIR)"
 	cd "$(INSTALL_BINDIR)"; \
 	  $(LN) flexlink.opt$(EXE) flexlink$(EXE)
@@ -1112,7 +1124,7 @@ distclean: clean
 	      boot/*.cm* boot/libcamlrun.a boot/libcamlrun.lib boot/ocamlc.opt
 	rm -f Makefile.config Makefile.build_config
 	rm -f runtime/caml/m.h runtime/caml/s.h
-	rm -rf autom4te.cache
+	rm -rf autom4te.cache flexdll-sources
 	rm -f config.log config.status libtool
 	rm -f tools/eventlog_metadata
 	rm -f tools/*.bak

Makefile.build_config.in

@@ -38,3 +38,8 @@ RLWRAP=@rlwrap@
 # Which document generator: odoc or ocamldoc?
 DOCUMENTATION_TOOL=@documentation_tool@
 DOCUMENTATION_TOOL_CMD=@documentation_tool_cmd@
+
+# The location of the FlexDLL sources to use (usually provided as the flexdll
+# Git submodule)
+FLEXDLL_SOURCES=@flexdir@
+BOOTSTRAPPING_FLEXDLL=@bootstrapping_flexdll@

Makefile.common

@@ -57,7 +57,7 @@ else
   BOOT_OCAMLC = $(OCAMLRUN) $(ROOTDIR)/boot/ocamlc
 endif
 
-ifeq "$(FLEXDLL_SUBMODULE_PRESENT)" ""
+ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
   FLEXLINK_ENV =
   CAMLOPT_CMD = $(CAMLOPT)
   OCAMLOPT_CMD = $(OCAMLOPT)
@@ -73,7 +73,7 @@ else
   MKLIB_CMD = $(FLEXLINK_ENV) $(MKLIB)
   ocamlc_cmd = $(FLEXLINK_ENV) $(ocamlc)
   ocamlopt_cmd = $(FLEXLINK_ENV) $(ocamlopt)
-endif
+endif # ifeq "$(BOOTSTRAPPING_FLEXDLL)" "false"
 
 OPTCOMPFLAGS=
 ifeq "$(FUNCTION_SECTIONS)" "true"

Makefile.config.in

@@ -203,7 +203,6 @@ OCAMLOPT_CPPFLAGS=@ocamlc_cppflags@
 NATIVECCLIBS=@nativecclibs@
 SYSTHREAD_SUPPORT=@systhread_support@
 PACKLD=@PACKLD@
-IFLEXDIR=@iflexdir@
 FLEXDLL_CHAIN=@flexdll_chain@
 EXTRALIBS=@extralibs@
 CCOMPTYPE=@ccomptype@

README.win32.adoc

@@ -76,7 +76,7 @@ bootstrap FlexDLL, you will need to ensure that the directory to which you
 install FlexDLL is included in your `PATH` environment variable. Note: binary
 distributions of FlexDLL are compatible only with Visual Studio 2013 and
 earlier; for Visual Studio 2015 and later, you will need to compile the C
-objects from source, or bootstrap FlexDLL (see below).
+objects from source, or configure ocaml with the `--with-flexdll` option.
 
 The base bytecode system (ocamlc, ocaml, ocamllex, ocamlyacc, ...) of all three
 ports runs without any additional tools.
@@ -246,7 +246,7 @@ C code (`ocamlc -custom`) require the appropriate Mingw-w64 gcc and the
 `mingw64-i686-gcc-core` package for 32-bit and the `mingw64-x86_64-gcc-core`
 package for 64-bit.
 
-  - Do not try to use the Cygwin version of flexdll for this port.
+  - The Cygwin version of flexdll does not work with this port.
 
   - The standalone mingw toolchain from the Mingw-w64 project
     (http://mingw-w64.org/) is not supported. Please use the version packaged in

aclocal.m4

@@ -130,6 +130,7 @@ AC_DEFUN([OCAML_CC_SAVE_VARIABLES], [
   saved_CC="$CC"
   saved_CFLAGS="$CFLAGS"
   saved_CPPFLAGS="$CPPFLAGS"
+  saved_LIBS="$LIBS"
   saved_ac_ext="$ac_ext"
   saved_ac_compile="$ac_compile"
   # Move the content of confdefs.h to another file so it does not
@@ -147,6 +148,7 @@ AC_DEFUN([OCAML_CC_RESTORE_VARIABLES], [
   CPPFLAGS="$saved_CPPFLAGS"
   CFLAGS="$saved_CFLAGS"
   CC="$saved_CC"
+  LIBS="$saved_LIBS"
 ])
 
 AC_DEFUN([OCAML_AS_HAS_DEBUG_PREFIX_MAP], [
@@ -290,3 +292,67 @@ AC_DEFUN([OCAML_CHECK_LIBUNWIND], [
   LDFLAGS="$SAVED_LDFLAGS"
   CFLAGS="$SAVED_CFLAGS"
 ])
+
+AC_DEFUN([OCAML_TEST_FLEXLINK], [
+  OCAML_CC_SAVE_VARIABLES
+
+  AC_MSG_CHECKING([whether $1 works])
+
+  AC_COMPILE_IFELSE(
+    [AC_LANG_SOURCE([int answer = 42;])],
+    [# Create conftest1.$ac_objext as a symlink on Cygwin to ensure that native
+    # flexlink can cope. The reverse test is unnecessary (a Cygwin-compiled
+    # flexlink can read anything).
+    mv conftest.$ac_objext conftest1.$ac_objext
+    AS_CASE([$4],[*-pc-cygwin],
+      [ln -s conftest1.$ac_objext conftest2.$ac_objext],
+      [cp conftest1.$ac_objext conftest2.$ac_objext])
+
+    CC="$1 -chain $2 -exe"
+    LIBS="conftest2.$ac_objext"
+    CPPFLAGS="$3 $CPPFLAGS"
+    AC_LINK_IFELSE(
+      [AC_LANG_SOURCE([int main() { return 0; }])],
+      [AC_MSG_RESULT([yes])],
+      [AC_MSG_RESULT([no])
+      AC_MSG_ERROR([$1 does not work])])],
+    [AC_MSG_RESULT([unexpected compile error])
+    AC_MSG_ERROR([error calling the C compiler])])
+
+  OCAML_CC_RESTORE_VARIABLES
+])
+
+AC_DEFUN([OCAML_TEST_FLEXDLL_H], [
+  OCAML_CC_SAVE_VARIABLES
+
+  AS_IF([test -n "$1"],[CPPFLAGS="-I $1 $CPPFLAGS"])
+  have_flexdll_h=no
+  AC_CHECK_HEADER([flexdll.h],[have_flexdll_h=yes],[have_flexdll_h=no])
+  AS_IF([test x"$have_flexdll_h" = 'xno'],
+    [AS_IF([test -n "$1"],
+      [AC_MSG_ERROR([$1/flexdll.h appears unusable])])])
+
+  OCAML_CC_RESTORE_VARIABLES
+])
+
+AC_DEFUN([OCAML_TEST_FLEXLINK_WHERE], [
+  OCAML_CC_SAVE_VARIABLES
+
+  AC_MSG_CHECKING([if "$1 -where" includes flexdll.h])
+  flexlink_where="$($1 -where | tr -d '\r')"
+  CPPFLAGS="$CPPFLAGS -I \"$flexlink_where\""
+  cat > conftest.c <<"EOF"
+#include <flexdll.h>
+int main (void) {return 0;}
+EOF
+  cat > conftest.Makefile <<EOF
+all:
+	$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.c $LIBS
+EOF
+  AS_IF([make -f conftest.Makefile >/dev/null 2>/dev/null],
+    [have_flexdll_h=yes
+    AC_MSG_RESULT([yes])],
+    [AC_MSG_RESULT([no])])
+
+  OCAML_CC_RESTORE_VARIABLES
+])