feat(pkg): use the relocatable compiler

doc/changes/added/14357.md

@@ -0,0 +1,2 @@
+- Enable the relocatable compiler by default for package management (#14357,
+  fixes #14012, @Alizter)

doc/explanation/package-management.md

@@ -137,16 +137,21 @@ However, it is also possible to declare specific revisions of the repositories,
 to get a reproducible solution. Due to using Git, any previous revision of the
 repository can be used by specifying a commit hash.
 
-Dune uses two repositories by default:
+Dune uses three repositories by default, in order of priority:
 
-* `upstream` refers to the default branch of `opam-repository`, which contains
-  all the publicly released packages.
 * `overlay` refers to
   [opam-overlays](https://github.com/ocaml-dune/opam-overlays), which defines
   packages patched to work with package management. The long-term goal is to
   have as few packages as possible in this repository as more and more packages
   work within Dune Package Management upstream. Check the
   [compatibility](#compatibility) section for details.
+* `relocatable` refers to the `relocatable` branch of
+  [dra27/opam-repository](https://github.com/dra27/opam-repository/tree/relocatable),
+  which provides a relocatable version of the OCaml compiler. This allows the
+  compiler to be built and cached independently of the project's build
+  directory.
+* `upstream` refers to the default branch of `opam-repository`, which contains
+  all the publicly released packages.
 
 #### Solving
 

src/dune_digest/digest.ml

@@ -104,28 +104,36 @@ let file file =
   digest_and_close_fd fd
 ;;
 
+(* Throttle concurrent [file_async] calls so an unbounded [parallel_map] over
+   many targets (e.g. the relocatable compiler) does not exhaust the process
+   fd limit. 100 sits comfortably above the background thread pool's worker
+   count (so digesting never starves it) while staying well under the typical
+   1024 fd soft limit; raising that limit is hazardous because some code
+   still falls back to [select()], which has a hard FD_SETSIZE (1024) cap. *)
+let digest_throttle = lazy (Fiber.Throttle.create 100)
 let async_digest_minimum = 1_000
 
 let file_async file =
-  let open Fiber.O in
-  let* () = Fiber.return () in
-  let fd = open_for_digest file in
-  Counter.incr Metrics.Digest.File.count;
-  let size =
-    match Unix.fstat (Fd.unsafe_to_unix_file_descr fd) with
-    | exception exn ->
-      Fd.close fd;
-      raise exn
-    | stat -> stat.st_size
-  in
-  Counter.add Metrics.Digest.File.bytes size;
-  if size = 0
-  then
-    let+ () = Fiber.return @@ Fd.close fd in
-    Lazy.force zero
-  else if size < async_digest_minimum
-  then Fiber.return (digest_and_close_fd fd)
-  else Dune_scheduler.Scheduler.async_exn (fun () -> digest_and_close_fd fd)
+  Fiber.Throttle.run (Lazy.force digest_throttle) ~f:(fun () ->
+    let open Fiber.O in
+    let* () = Fiber.return () in
+    let fd = open_for_digest file in
+    Counter.incr Metrics.Digest.File.count;
+    let size =
+      match Unix.fstat (Fd.unsafe_to_unix_file_descr fd) with
+      | exception exn ->
+        Fd.close fd;
+        raise exn
+      | stat -> stat.st_size
+    in
+    Counter.add Metrics.Digest.File.bytes size;
+    if size = 0
+    then
+      let+ () = Fiber.return @@ Fd.close fd in
+      Lazy.force zero
+    else if size < async_digest_minimum
+    then Fiber.return (digest_and_close_fd fd)
+    else Dune_scheduler.Scheduler.async_exn (fun () -> digest_and_close_fd fd))
 ;;
 
 let equal = Blake3_mini.Digest.equal

src/dune_digest/digest.mli

@@ -108,5 +108,7 @@ val path_with_stats_async
   -> (t, Path_digest_error.t) result Fiber.t
 
 (** Digest a file taking the [executable] bit into account. Should not be called
-    on a directory. *)
+    on a directory. Digesting is done in the background thread pool, with the
+    number of concurrent calls capped by a global throttle so that we do not
+    exceed the process's open file descriptor limit. *)
 val file_with_executable_bit : executable:bool -> Path.t -> t Fiber.t

src/dune_pkg/dev_tool.ml

@@ -128,7 +128,12 @@ let needs_to_build_with_same_compiler_as_project = function
 let compiler_package_names =
   List.map
     ~f:Package_name.of_string
-    [ "ocaml"; "ocaml-base-compiler"; "ocaml-variants"; "ocaml-compiler" ]
+    [ "ocaml"
+    ; "ocaml-base-compiler"
+    ; "ocaml-variants"
+    ; "ocaml-compiler"
+    ; "relocatable-compiler"
+    ]
 ;;
 
 let is_compiler_package name =

src/dune_pkg/workspace.ml

@@ -51,6 +51,16 @@ module Repository = struct
     }
   ;;
 
+  let relocatable =
+    { name = "relocatable"
+    ; url =
+        ( Loc.none
+        , OpamUrl.of_string
+            "git+https://github.com/ocaml-dune/opam-repository-relocatable.git#relocatable"
+        )
+    }
+  ;;
+
   let binary_packages =
     { name = "binary-packages"
     ; url =

src/dune_pkg/workspace.mli

@@ -9,6 +9,7 @@ module Repository : sig
   val equal : t -> t -> bool
   val upstream : t
   val overlay : t
+  val relocatable : t
   val binary_packages : t
   val decode : t Decoder.t
 

src/source/workspace.ml

@@ -13,7 +13,9 @@ module Pin_stanza = Dune_lang.Pin_stanza
 module Repository = Dune_pkg.Pkg_workspace.Repository
 module Solver_env = Dune_pkg.Solver_env
 
-let default_repositories = [ Repository.overlay; Repository.upstream ]
+let default_repositories =
+  [ Repository.overlay; Repository.relocatable; Repository.upstream ]
+;;
 
 module Lock_dir = struct
   type t =