Dune upgrades to 3.9.2 but fails to build other packages (Mint, ecryptfs)

[alan-j-hu]

Expected Behavior

opam upgrade -v successfully upgrades my packages.

Actual Behavior

The build of packages using Dune fails.

Reproduction

opam update -v

I can still revert to the previous opam state, which has Dune 3.7.1.

Specifications

OCaml 5, Dune 3.9.2, Linux Mint 21.2, Linux kernel 5.15.0-78-generic, ecryptfs

Additional information

This is the output of the opam upgrade -v command, there are a lot of errors that are hard for me to make sense of: https://gist.github.com/alan-j-hu/7bf8fcc6ede90c6ebe604af38745334b

This is probably still related to #8041 and #8210.

[alan-j-hu]

When I cloned Dune and ran make dev, I got the following errors:

ocamlc -output-complete-exe -w -24 -g -o .duneboot.exe -I boot -I +unix unix.cma boot/libs.ml boot/duneboot.ml
./.duneboot.exe
./_boot/dune.exe build @install
File "otherlibs/dune-rpc/private/dbus_address.mll", line 1, character 0: syntax error.
File "otherlibs/dune-glob/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/ocamlc-loc/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/configurator/src/extract_obj.mll", line 1, character 0: syntax error.
File "otherlibs/dune-private-libs/meta_parser/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/versioned_file_first_line.mll", line 1, character 0: syntax error.
File "src/dune_lang/dune_file_script.mll", line 1, character 0: syntax error.
File "src/dune_rules/ocamlobjinfo.mll", line 1, character 0: syntax error.
File "src/dune_rules/cram/cram_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/findlib/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/dune_file.ml", line 1929, characters 27-27:
Error: Syntax error: ')' expected
File "src/dune_rules/dune_file.ml", line 1928, characters 14-15:
1928 |               (Syntax.since Stanza.syntax (3, 9)
                     ^
  This '(' might be unmatched
File "src/dune_rules/dune", line 62, characters 0-273:
62 | (rule
63 |  (mode promote)
64 |  (alias runtest)
....
72 |     DUNE_CONFIGURE_OUTPUT
73 |     "src/dune_rules/setup.defaults.ml"
74 |     (run %{ocaml} %{configure})))))
Error: Rule failed to generate the following targets:
- src/dune_rules/setup.defaults.ml
File "vendor/opam-file-format/opamBaseParser.mly", line 0: unexpected end-of-file
File "_build/default/vendor/sha/c_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam-file-format/opamLexer.mll", line 1, character 0: syntax error.
File "_build/default/otherlibs/xdg/c_library_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "_build/default/src/fsevents/flags/sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam/src/format/opamInterpLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamLineLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamFile.ml", line 3893, characters 7-7:
Error: Syntax error: ']' expected
File "vendor/opam/src/format/opamFile.ml", line 3856, characters 4-5:
3856 |     [
           ^
  This '[' might be unmatched
File "_build/default/otherlibs/configurator/src/flags/flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
make: *** [Makefile:56: dev] Error 1  

which resemble the errors I got when Dune 3.9.2 was building the other packages.

[alan-j-hu]

I did a little print_endline debugging. When I make the following change to io.ml:

diff --git a/otherlibs/stdune/src/io.ml b/otherlibs/stdune/src/io.ml
index 48cbc3e09..68b482e5e 100644
--- a/otherlibs/stdune/src/io.ml
+++ b/otherlibs/stdune/src/io.ml
@@ -136,16 +136,20 @@ module Copyfile = struct
          Eventually, we should stop using exceptions for signalling these
          errors. But that's a bit of a large change since there's a lot of
          exception catching to audit. *)
+      print_endline "Entering";
       match setup_copy ?chmod ~src ~dst () with
-      | Error (`Exn exn) -> Exn_with_backtrace.reraise exn
-      | Error `Src_is_a_dir -> raise (Sys_error "Is a directory")
+      | Error (`Exn exn) -> print_endline "A"; Exn_with_backtrace.reraise exn
+      | Error `Src_is_a_dir -> print_endline "B"; raise (Sys_error "Is a directory")
       | Error `Dst_is_a_dir ->
+        print_endline "C";
         let message = Printf.sprintf "%s: Is a directory" dst in
         raise (Sys_error message)
       | Error `Src_missing ->
+        print_endline "D";
         let message = Printf.sprintf "%s: No such file or directory" src in
         raise (Sys_error message)
       | Ok (src, dst, src_size) ->
+        print_endline "E";
         Exn.protect
           ~f:(fun () ->
             try sendfile ~src ~dst src_size

and then run make dev from scratch, I get the following output:

ocamlc -output-complete-exe -w -24 -g -o .duneboot.exe -I boot -I +unix unix.cma boot/libs.ml boot/duneboot.ml
./.duneboot.exe
./_boot/dune.exe build @install
Done: 5% (67/1279, 1212 left) (jobs: 0)Entering
E
File "otherlibs/dune-rpc/private/dbus_address.mll", line 1, character 0: syntax error.
File "otherlibs/dune-glob/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/ocamlc-loc/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/configurator/src/extract_obj.mll", line 1, character 0: syntax error.
File "otherlibs/dune-private-libs/meta_parser/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/versioned_file_first_line.mll", line 1, character 0: syntax error.
File "src/dune_lang/dune_file_script.mll", line 1, character 0: syntax error.
File "src/dune_rules/ocamlobjinfo.mll", line 1, character 0: syntax error.
File "src/dune_rules/cram/cram_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/findlib/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/dune_file.ml", line 1929, characters 27-27:
Error: Syntax error: ')' expected
File "src/dune_rules/dune_file.ml", line 1928, characters 14-15:
1928 |               (Syntax.since Stanza.syntax (3, 9)
                     ^
  This '(' might be unmatched
File "src/dune_rules/dune", line 62, characters 0-273:
62 | (rule
63 |  (mode promote)
64 |  (alias runtest)
....
72 |     DUNE_CONFIGURE_OUTPUT
73 |     "src/dune_rules/setup.defaults.ml"
74 |     (run %{ocaml} %{configure})))))
Error: Rule failed to generate the following targets:
- src/dune_rules/setup.defaults.ml
File "_build/default/otherlibs/xdg/c_library_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "_build/default/vendor/sha/c_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam-file-format/opamBaseParser.mly", line 0: unexpected end-of-file
File "vendor/opam-file-format/opamLexer.mll", line 1, character 0: syntax error.
File "_build/default/src/fsevents/flags/sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "_build/default/otherlibs/configurator/src/flags/flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam/src/format/opamInterpLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamLineLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamFile.ml", line 3893, characters 7-7:
Error: Syntax error: ']' expected
File "vendor/opam/src/format/opamFile.ml", line 3856, characters 4-5:
3856 |     [
           ^
  This '[' might be unmatched
make: *** [Makefile:56: dev] Error 1  

So, the code reaches the path I labeled "E" (and apparently didn't touch the other paths), and then fails.

Edit: with the following diff:

diff --git a/otherlibs/stdune/src/io.ml b/otherlibs/stdune/src/io.ml
index 48cbc3e09..3a5a03af3 100644
--- a/otherlibs/stdune/src/io.ml
+++ b/otherlibs/stdune/src/io.ml
@@ -136,20 +136,25 @@ module Copyfile = struct
          Eventually, we should stop using exceptions for signalling these
          errors. But that's a bit of a large change since there's a lot of
          exception catching to audit. *)
+      print_endline "Entering";
       match setup_copy ?chmod ~src ~dst () with
-      | Error (`Exn exn) -> Exn_with_backtrace.reraise exn
-      | Error `Src_is_a_dir -> raise (Sys_error "Is a directory")
+      | Error (`Exn exn) -> print_endline "A"; Exn_with_backtrace.reraise exn
+      | Error `Src_is_a_dir -> print_endline "B"; raise (Sys_error "Is a directory")
       | Error `Dst_is_a_dir ->
+        print_endline "C";
         let message = Printf.sprintf "%s: Is a directory" dst in
         raise (Sys_error message)
       | Error `Src_missing ->
+        print_endline "D";
         let message = Printf.sprintf "%s: No such file or directory" src in
         raise (Sys_error message)
       | Ok (src, dst, src_size) ->
+        print_endline "E";
         Exn.protect
           ~f:(fun () ->
-            try sendfile ~src ~dst src_size
+            try print_endline "E1"; sendfile ~src ~dst src_size
             with Unix.Unix_error (EINVAL, "sendfile", _) ->
+              print_endline "E2";
               let ic = Unix.in_channel_of_descr src in
               let oc = Unix.out_channel_of_descr dst in
               copy_channels ic oc)

I get:

ocamlc -output-complete-exe -w -24 -g -o .duneboot.exe -I boot -I +unix unix.cma boot/libs.ml boot/duneboot.ml
./.duneboot.exe
./_boot/dune.exe build @install
Done: 5% (67/1279, 1212 left) (jobs: 0)Entering
E
E1
E2
File "otherlibs/dune-rpc/private/dbus_address.mll", line 1, character 0: syntax error.
File "otherlibs/dune-glob/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/ocamlc-loc/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/configurator/src/extract_obj.mll", line 1, character 0: syntax error.
File "otherlibs/dune-private-libs/meta_parser/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/versioned_file_first_line.mll", line 1, character 0: syntax error.
File "src/dune_lang/dune_file_script.mll", line 1, character 0: syntax error.
File "src/dune_rules/ocamlobjinfo.mll", line 1, character 0: syntax error.
File "src/dune_rules/cram/cram_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/findlib/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/dune_file.ml", line 1929, characters 27-27:
Error: Syntax error: ')' expected
File "src/dune_rules/dune_file.ml", line 1928, characters 14-15:
1928 |               (Syntax.since Stanza.syntax (3, 9)
                     ^
  This '(' might be unmatched
File "src/dune_rules/dune", line 62, characters 0-273:
62 | (rule
63 |  (mode promote)
64 |  (alias runtest)
....
72 |     DUNE_CONFIGURE_OUTPUT
73 |     "src/dune_rules/setup.defaults.ml"
74 |     (run %{ocaml} %{configure})))))
Error: Rule failed to generate the following targets:
- src/dune_rules/setup.defaults.ml
File "_build/default/otherlibs/xdg/c_library_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "_build/default/vendor/sha/c_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam-file-format/opamBaseParser.mly", line 0: unexpected end-of-file
File "vendor/opam-file-format/opamLexer.mll", line 1, character 0: syntax error.
File "_build/default/src/fsevents/flags/sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "_build/default/otherlibs/configurator/src/flags/flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam/src/format/opamInterpLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamLineLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamFile.ml", line 3893, characters 7-7:
Error: Syntax error: ']' expected
File "vendor/opam/src/format/opamFile.ml", line 3856, characters 4-5:
3856 |     [
           ^
  This '[' might be unmatched
make: *** [Makefile:56: dev] Error 1 

meaning that copy_channels is probably buggy.

[rgrinberg]

@emillon two things I'm wondering about:

1. Do we need to seek input before the fallback?
2. Do we need to ftruncate the output before the fallback?

[alan-j-hu]

I realized I was not doing a clean build previously, as I did not remove the _build directory, only _boot. As a result, the above information (about when the sendfile_with_fallback function was called) was incorrect. With a clean build and the following change:

diff --git a/otherlibs/stdune/src/io.ml b/otherlibs/stdune/src/io.ml
index 48cbc3e09..8dd93dc14 100644
--- a/otherlibs/stdune/src/io.ml
+++ b/otherlibs/stdune/src/io.ml
@@ -67,6 +67,7 @@ let copy_channels =
     match input ic buf 0 buf_len with
     | 0 -> ()
     | n ->
+      output stderr buf 0 n;
       output oc buf 0 n;
       loop ic oc
   in
@@ -136,20 +137,25 @@ module Copyfile = struct
          Eventually, we should stop using exceptions for signalling these
          errors. But that's a bit of a large change since there's a lot of
          exception catching to audit. *)
+      prerr_endline "Entering";
       match setup_copy ?chmod ~src ~dst () with
-      | Error (`Exn exn) -> Exn_with_backtrace.reraise exn
-      | Error `Src_is_a_dir -> raise (Sys_error "Is a directory")
+      | Error (`Exn exn) -> print_endline "A"; Exn_with_backtrace.reraise exn
+      | Error `Src_is_a_dir -> print_endline "B"; raise (Sys_error "Is a directory")
       | Error `Dst_is_a_dir ->
+        prerr_endline "C";
         let message = Printf.sprintf "%s: Is a directory" dst in
         raise (Sys_error message)
       | Error `Src_missing ->
+        prerr_endline "D";
         let message = Printf.sprintf "%s: No such file or directory" src in
         raise (Sys_error message)
       | Ok (src, dst, src_size) ->
+        prerr_endline "E";
         Exn.protect
           ~f:(fun () ->
-            try sendfile ~src ~dst src_size
+            try prerr_endline "E1"; sendfile ~src ~dst src_size
             with Unix.Unix_error (EINVAL, "sendfile", _) ->
+              prerr_endline "E2";
               let ic = Unix.in_channel_of_descr src in
               let oc = Unix.out_channel_of_descr dst in
               copy_channels ic oc)

The stderr becomes https://gist.githubusercontent.com/alan-j-hu/3e3fa8d72e557fb8a07cee4119e67da1/raw/fa103276db6af27fba3762802b195ac4b9164483/make-output.txt.

From what I can tell, the files appear to be read correctly.

It is possible that the cause of the error is something other than sendfile_with_fallback.

[alan-j-hu]

Reverting to the old version of copy_file from the 3.7 branch like so:

diff --git a/otherlibs/stdune/src/io.ml b/otherlibs/stdune/src/io.ml
index 48cbc3e09..078b7dba2 100644
--- a/otherlibs/stdune/src/io.ml
+++ b/otherlibs/stdune/src/io.ml
@@ -181,7 +181,7 @@ module Copyfile = struct
     Exn.protectx (setup_copy ?chmod ~src ~dst ()) ~finally:close_both
       ~f:(fun (ic, oc) -> copy_channels ic oc)
 
-  let copy_file_best =
+  let _copy_file_best =
     match available with
     | `Sendfile -> sendfile_with_fallback
     | `Copyfile -> copyfile
@@ -190,9 +190,8 @@ module Copyfile = struct
   let copy_file_impl = ref `Best
 
   let copy_file ?chmod ~src ~dst () =
-    match !copy_file_impl with
-    | `Portable -> copy_file_portable ?chmod ~src ~dst ()
-    | `Best -> copy_file_best ?chmod ~src ~dst ()
+    Exn.protectx (setup_copy ?chmod ~src ~dst ()) ~finally:close_both
+      ~f:(fun (ic, oc) -> copy_channels ic oc)
 end
 
 let set_copy_impl m = Copyfile.copy_file_impl := m

and then building from scratch with rm -rf _boot _build and then make dev results in a successful build. So, I still believe the issue is with the new copy_file implementation.

When I build with the current main branch, the stderr is of course

ocamlc -output-complete-exe -w -24 -g -o .duneboot.exe -I boot -I +unix unix.cma boot/libs.ml boot/duneboot.ml
./.duneboot.exe
./_boot/dune.exe build @install
File "otherlibs/dune-rpc/private/dbus_address.mll", line 1, character 0: syntax error.
File "otherlibs/dune-glob/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/ocamlc-loc/src/lexer.mll", line 1, character 0: syntax error.
File "otherlibs/configurator/src/extract_obj.mll", line 1, character 0: syntax error.
File "otherlibs/dune-private-libs/meta_parser/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/lexer.mll", line 1, character 0: syntax error.
File "src/dune_sexp/versioned_file_first_line.mll", line 1, character 0: syntax error.
File "src/dune_lang/dune_file_script.mll", line 1, character 0: syntax error.
File "src/dune_rules/ocamlobjinfo.mll", line 1, character 0: syntax error.
File "src/dune_rules/cram/cram_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/findlib/meta_lexer.mll", line 1, character 0: syntax error.
File "src/dune_rules/dune_file.ml", line 1929, characters 27-27:
Error: Syntax error: ')' expected
File "src/dune_rules/dune_file.ml", line 1928, characters 14-15:
1928 |               (Syntax.since Stanza.syntax (3, 9)
                     ^
  This '(' might be unmatched
File "src/dune_rules/dune", line 62, characters 0-273:
62 | (rule
63 |  (mode promote)
64 |  (alias runtest)
....
72 |     DUNE_CONFIGURE_OUTPUT
73 |     "src/dune_rules/setup.defaults.ml"
74 |     (run %{ocaml} %{configure})))))
Error: Rule failed to generate the following targets:
- src/dune_rules/setup.defaults.ml
File "_build/default/vendor/sha/c_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam-file-format/opamBaseParser.mly", line 0: unexpected end-of-file
File "vendor/opam-file-format/opamLexer.mll", line 1, character 0: syntax error.
File "_build/default/otherlibs/xdg/c_library_flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam/src/format/opamInterpLexer.mll", line 1, character 0: syntax error.
File "vendor/opam/src/format/opamLineLexer.mll", line 1, character 0: syntax error.
File "_build/default/src/fsevents/flags/sexp", line 1, characters 0-0:
Error: no s-expression found in input
File "vendor/opam/src/format/opamFile.ml", line 3893, characters 7-7:
Error: Syntax error: ']' expected
File "vendor/opam/src/format/opamFile.ml", line 3856, characters 4-5:
3856 |     [
           ^
  This '[' might be unmatched
File "_build/default/otherlibs/configurator/src/flags/flags.sexp", line 1, characters 0-0:
Error: no s-expression found in input
make: *** [Makefile:56: dev] Error 1  

I checked _build/default/src/fsevents/flags/sexp and it is an empty file, and I checked _build/vendor/opam/src/format/opamFile.ml and it is indeed truncated/incomplete.

[alan-j-hu]

I did some investigation. The following code results in the error:

diff --git a/otherlibs/stdune/src/io.ml b/otherlibs/stdune/src/io.ml
index 48cbc3e09..301e0a8ae 100644
--- a/otherlibs/stdune/src/io.ml
+++ b/otherlibs/stdune/src/io.ml
@@ -92,7 +92,7 @@ module Copyfile = struct
      it fails for w/e reason *)
   external copyfile : string -> string -> unit = "stdune_copyfile"
 
-  external sendfile : src:Unix.file_descr -> dst:Unix.file_descr -> int -> unit
+  external _sendfile : src:Unix.file_descr -> dst:Unix.file_descr -> int -> unit
     = "stdune_sendfile"
 
   let available =
@@ -145,17 +145,10 @@ module Copyfile = struct
       | Error `Src_missing ->
         let message = Printf.sprintf "%s: No such file or directory" src in
         raise (Sys_error message)
-      | Ok (src, dst, src_size) ->
-        Exn.protect
-          ~f:(fun () ->
-            try sendfile ~src ~dst src_size
-            with Unix.Unix_error (EINVAL, "sendfile", _) ->
-              let ic = Unix.in_channel_of_descr src in
-              let oc = Unix.out_channel_of_descr dst in
-              copy_channels ic oc)
-          ~finally:(fun () ->
-            Unix.close src;
-            Unix.close dst)
+      | Ok (src, dst, _src_size) ->
+        Exn.protectx (Unix.in_channel_of_descr src, Unix.out_channel_of_descr dst)
+          ~finally:(fun (_, _) -> Unix.close src; Unix.close dst)
+          ~f:(fun (ic, oc) -> copy_channels ic oc)
 
   let copyfile ?chmod ~src ~dst () =
     let src_stats =

However, the following code works fine:

diff --git a/otherlibs/stdune/src/io.ml b/otherlibs/stdune/src/io.ml
index 48cbc3e09..383d29753 100644
--- a/otherlibs/stdune/src/io.ml
+++ b/otherlibs/stdune/src/io.ml
@@ -92,7 +92,7 @@ module Copyfile = struct
      it fails for w/e reason *)
   external copyfile : string -> string -> unit = "stdune_copyfile"
 
-  external sendfile : src:Unix.file_descr -> dst:Unix.file_descr -> int -> unit
+  external _sendfile : src:Unix.file_descr -> dst:Unix.file_descr -> int -> unit
     = "stdune_sendfile"
 
   let available =
@@ -145,17 +145,10 @@ module Copyfile = struct
       | Error `Src_missing ->
         let message = Printf.sprintf "%s: No such file or directory" src in
         raise (Sys_error message)
-      | Ok (src, dst, src_size) ->
-        Exn.protect
-          ~f:(fun () ->
-            try sendfile ~src ~dst src_size
-            with Unix.Unix_error (EINVAL, "sendfile", _) ->
-              let ic = Unix.in_channel_of_descr src in
-              let oc = Unix.out_channel_of_descr dst in
-              copy_channels ic oc)
-          ~finally:(fun () ->
-            Unix.close src;
-            Unix.close dst)
+      | Ok (src, dst, _src_size) ->
+        Exn.protectx (Unix.in_channel_of_descr src, Unix.out_channel_of_descr dst)
+          ~finally:close_both
+          ~f:(fun (ic, oc) -> copy_channels ic oc)
 
   let copyfile ?chmod ~src ~dst () =
     let src_stats =

So, Unix.close src; Unix.close dst must not be doing the same thing as close_both, and the former is causing issues.

[alan-j-hu]

I've figured out the issue. OCaml channels are buffered. Stdlib.close_out does not only close the file descriptor, it also flushes the buffer.

The cleanup code overlooks this fact and assumes that Unix.close dst is all that is necessary even if the channel-based fallback path was taken. Because the buffer was not flushed, some long files were copied incompletely. The sendfile path and the channel fallback path require different cleanup code because channels are buffered.