Add workspace feature (#3841)

This feature allows one to define a "workspace" in a project file by adding

```toml
[workspace]
projects = ["packageA", "docs", "test"]
```

to it. All projects in a workspace are resolved together into a single manifest file in the "root" project (the project that is not included by any other project in a workspace). The compat of all dependencies are intersected among the projects.

Currently, every project in a workspace must be in a folder at the same level as the project that includes it into the workspace.

If the test folder is included in the workspace, running `Pkg.test` is similar to just activating the test project and including the `runtest.jl` file.

Many functions now take the `workspace` argument to decide if the function should operate on the active project or on the full workspace:
- `Pkg.precompile`
- `Pkg.status`
- `Pkg.why`
- `Pkg.instantiate`

Author: KristofferC

.github/workflows/test.yml

@@ -96,7 +96,7 @@ jobs:
           version: 'nightly'
       - name: Generate docs
         run: |
-          julia --project --color=yes -e 'using Pkg; Pkg.activate("docs"); Pkg.instantiate(); Pkg.develop(PackageSpec(path = pwd()))'
+          julia --project --color=yes -e 'using Pkg; Pkg.activate("docs"); Pkg.instantiate();'
           julia --project=docs --color=yes docs/make.jl pdf
         env:
           DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}

CHANGELOG.md

@@ -3,6 +3,9 @@ Pkg v1.12 Release Notes
 
 - It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml.
   This can be used to add non-registered normal or test dependencies.
+- Pkg now has support for "workspaces" which is a way to resolve multiple project files into a single manifest.
+  The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated
+  to take a `workspace` option. Read more about this feature in the manual about the TOML-files.
 
 Pkg v1.11 Release Notes
 =======================

Project.toml

@@ -5,6 +5,9 @@ license = "MIT"
 desc = "The next-generation Julia package manager."
 version = "1.12.0"
 
+[workspace]
+projects = ["test", "docs"]
+
 [deps]
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -27,14 +30,3 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [extensions]
 REPLExt = "REPL"
-
-[compat]
-HistoricalStdlibVersions = "1.2"
-
-[extras]
-HistoricalStdlibVersions = "6df8b67a-e8a0-4029-b4b7-ac196fe72102"
-Preferences = "21216c6a-2e73-6563-6e65-726566657250"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["REPL", "Test", "Preferences", "HistoricalStdlibVersions"]

docs/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [compat]
 Documenter = "1"

docs/src/toml-files.md

@@ -128,6 +128,24 @@ constraints in detail. It is also possible to list constraints on `julia` itself
 julia = "1.1"
 ```
 
+### The `[workspace]` section
+
+A project file can define a workspace by giving a set of projects that is part of that workspace.
+Each project in a workspace can include their own dependencies, compatibility information, and even function as full packages.
+
+When the package manager resolves dependencies, it considers the requirements of all the projects in the workspace. The compatible versions identified during this process are recorded in a single manifest file located next to the base project file.
+
+A workspace is defined in the base project by giving a list of the projects in it:
+
+```toml
+[workspace]
+projects = ["test", "docs", "benchmarks", "PrivatePackage"]
+```
+
+This structure is particularly beneficial for developers using a monorepo approach, where a large number of unregistered packages may be involved. It's also useful for adding documentation or benchmarks to a package by including additional dependencies beyond those of the package itself.
+
+Workspace can be nested, that ism a project that itself defines a workspace can also be part of another workspace.
+In this case, the workspaces are "merged" with a single manifest being stored alongside the "root project" (the project that doesn't have another workspace including it).
 
 ## `Manifest.toml`
 

ext/REPLExt/REPLExt.jl

@@ -68,10 +68,17 @@ function promptf()
         else
             project_name = projname(project_file)
             if project_name !== nothing
-                if textwidth(project_name) > 30
-                    project_name = first(project_name, 27) * "..."
+                root = Types.find_root_base_project(project_file)
+                rootname = projname(root)
+                if root !== project_file
+                    path_prefix = "/" * dirname(Types.relative_project_path(root, project_file))
+                else
+                    path_prefix = ""
+                end
+                if textwidth(rootname) > 30
+                    rootname = first(rootname, 27) * "..."
                 end
-                prefix = "($(project_name)) "
+                prefix = "($(rootname)$(path_prefix)) "
                 prev_prefix = prefix
                 prev_project_timestamp = mtime(project_file)
                 prev_project_file = project_file

src/API.jl

@@ -82,7 +82,7 @@ end
 
 dependencies() = dependencies(EnvCache())
 function dependencies(env::EnvCache)
-    pkgs = Operations.load_all_deps(env)
+    pkgs = Operations.load_all_deps_loadable(env)
     return Dict(pkg.uuid::UUID => package_info(env, pkg) for pkg in pkgs)
 end
 function dependencies(fn::Function, uuid::UUID)
@@ -1132,7 +1132,7 @@ end
 function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool=false,
                     strict::Bool=false, warn_loaded = true, already_instantiated = false, timing::Bool = false,
                     _from_loading::Bool=false, configs::Union{Base.Precompilation.Config,Vector{Base.Precompilation.Config}}=(``=>Base.CacheFlags()),
-                    kwargs...)
+                    workspace::Bool=false, kwargs...)
     Context!(ctx; kwargs...)
     if !already_instantiated
         instantiate(ctx; allow_autoprecomp=false, kwargs...)
@@ -1148,14 +1148,13 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
     io = ctx.io
     if io isa UnstableIO
         # precompile does quite a bit of output and using the UnstableIO can cause
-        # some slowdowns, the important part here is to not specialize the whole
-        # precompile function on the io
+        # some slowdowns
         io = io.io
     end
 
     activate(dirname(ctx.env.project_file)) do
         pkgs_name = String[pkg.name for pkg in pkgs]
-        return Base.Precompilation.precompilepkgs(pkgs_name; internal_call, strict, warn_loaded, timing, _from_loading, configs, io)
+        return Base.Precompilation.precompilepkgs(pkgs_name; internal_call, strict, warn_loaded, timing, _from_loading, configs, manifest=workspace, io)
     end
 end
 
@@ -1171,7 +1170,8 @@ end
 instantiate(; kwargs...) = instantiate(Context(); kwargs...)
 function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
                      update_registry::Bool=true, verbose::Bool=false,
-                     platform::AbstractPlatform=HostPlatform(), allow_build::Bool=true, allow_autoprecomp::Bool=true, kwargs...)
+                     platform::AbstractPlatform=HostPlatform(), allow_build::Bool=true, allow_autoprecomp::Bool=true,
+                     workspace::Bool=false, kwargs...)
     Context!(ctx; kwargs...)
     if Registry.download_default_registries(ctx.io)
         copy!(ctx.registries, Registry.reachable_registries())
@@ -1216,12 +1216,16 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
                  " Finally, run `Pkg.instantiate()` again.")
     end
     # check if all source code and artifacts are downloaded to exit early
-    if Operations.is_instantiated(ctx.env; platform)
+    if Operations.is_instantiated(ctx.env, workspace; platform)
         allow_autoprecomp && Pkg._auto_precompile(ctx, already_instantiated = true)
         return
     end
 
-    pkgs = Operations.load_all_deps(ctx.env)
+    if workspace
+        pkgs = Operations.load_all_deps(ctx.env)
+    else
+        pkgs = Operations.load_all_deps_loadable(ctx.env)
+    end
     try
         # First try without updating the registry
         Operations.check_registered(ctx.registries, pkgs)
@@ -1279,14 +1283,14 @@ end
 
 @deprecate status(mode::PackageMode) status(mode=mode)
 
-function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, extensions::Bool=false, io::IO=stdout_f())
+function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, workspace::Bool=false, outdated::Bool=false, compat::Bool=false, extensions::Bool=false, io::IO=stdout_f())
     if compat
         diff && pkgerror("Compat status has no `diff` mode")
         outdated && pkgerror("Compat status has no `outdated` mode")
         extensions && pkgerror("Compat status has no `extensions` mode")
         Operations.print_compat(ctx, pkgs; io)
     else
-        Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, extensions)
+        Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, extensions, workspace)
     end
     return nothing
 end
@@ -1419,7 +1423,7 @@ compat(;kwargs...) = compat(Context(); kwargs...)
 # why #
 #######
 
-function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, kwargs...)
+function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, workspace::Bool=false, kwargs...)
     require_not_empty(pkgs, :why)
 
     manifest_resolve!(ctx.env.manifest, pkgs)
@@ -1435,9 +1439,17 @@ function why(ctx::Context, pkgs::Vector{PackageSpec}; io::IO, kwargs...)
         end
     end
 
+    project_deps = Set(values(ctx.env.project.deps))
+
+    if workspace
+        for (_, project) in ctx.env.workspace
+            union!(project_deps, values(project.deps))
+        end
+    end
+
     function find_paths!(final_paths, current, path = UUID[])
         push!(path, current)
-        current in values(ctx.env.project.deps) && push!(final_paths, path) # record once we've traversed to a project dep
+        current in project_deps && push!(final_paths, path) # record once we've traversed to a project dep
         haskey(incoming, current) || return # but only return if we've reached a leaf that nothing depends on
         for p in incoming[current]
             if p in path