add support for glue packages

Author: KristofferC

docs/make.jl

@@ -35,6 +35,7 @@ makedocs(
         "managing-packages.md",
         "environments.md",
         "creating-packages.md",
+        "gluedeps.md",
         "compatibility.md",
         "registries.md",
         "artifacts.md",

docs/src/creating-packages.md

@@ -245,6 +245,81 @@ using Test
 Every dependency should in general have a compatibility constraint on it.
 This is an important topic so there is a separate chapter about it: [Compatibility](@ref Compatibility).
 
+## Conditional loading of code in packages (Glue packages)
+
+!!! note
+    This is a somewhat advanced section which can be skipped for people new to Julia and Julia packages.
+
+It is sometimes desirable to be able to extend some functionality of a package without having to
+unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency.
+A *glue package* is a file that gets automatically loaded when some other set of packages are
+loaded into the Julia session.
+
+A useful application of glue packages could be for a plotting package that should be able to plot
+objects from a wide variety of different Julia packages.
+Adding all those different Julia packages as dependencies
+could be expensive since they would end up getting loaded even if they were never used.
+Instead, these code required to plot objects for specific packages can be put into separate files
+(glue packages) which is only loaded when 
+
+Below is an example of how the code can be structured for a use case as outlined above.
+
+ `Project.toml`:
+ ```toml
+name = "Plotting"
+version = "0.1.0"
+uuid = "..."
+
+[deps]
+Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
+
+[gluedeps]
+Contour = "d38c429a-6771-53c6-b99e-75d170b6e991"
+
+[gluepkgs] 
+# name of glue package to the left
+# glue dependencies required to load the glue pkg to the right
+# use a list for multiple glue dependencies
+GlueContour = "Contour" 
+
+[compat] # compat can also be given on glue dependencies
+Colors = "0.12.8"
+Contour = "0.6.2"
+```
+
+`src/Plotting.jl`:
+```julia
+module Plotting
+using Colors
+
+function plot(x::Colors.Color)
+    # Some functionality for plotting a color here
+end
+
+end # module
+```
+
+`glue/GlueContour.jl` (can also be in `glue/GlueContour/GlueContour.jl`):
+```julia
+module GlueContour
+
+using Plotting, Contour
+
+function Plotting.plot(c::Contour.ContourCollection)
+    # Some functionality for plotting a contour here
+end
+
+end # module
+```
+
+A user that depends on `Plotting` will not pay the code of the "glue code" inside the `GlueContour` module.
+It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package will get loaded
+and provide the new functionality.
+
+Compatibility can be set on glue dependencies just like normal dependencies.
+
+A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself.
+
 ## Package naming guidelines
 
 Package names should be sensible to most Julia users, *even to those who are not domain experts*.

src/API.jl

@@ -154,7 +154,7 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why)
             pkgs = deepcopy(pkgs) # don't mutate input
             foreach(handle_package_input!, pkgs)
             ret = $f(ctx, pkgs; kwargs...)
-            $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx)
+            $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) # rm does too, but it's handled differently
             $(f in (:up, :pin, :free, :rm)) && Pkg._auto_gc(ctx)
             return ret
         end
@@ -301,9 +301,11 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode=PKGMODE_PROJECT, all_p
     ensure_resolved(ctx, ctx.env.manifest, pkgs)
 
     Operations.rm(ctx, pkgs; mode)
+
     return
 end
 
+
 function append_all_pkgs!(pkgs, ctx, mode)
     if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED
         for (name::String, uuid::UUID) in ctx.env.project.deps
@@ -1163,6 +1165,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
         isempty(depsmap) && pkgerror("No direct dependencies found matching $(repr(pkgs))")
     end
 
+    target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...")
+
     pkg_queue = Base.PkgId[]
     failed_deps = Dict{Base.PkgId, String}()
     skipped_deps = Base.PkgId[]
@@ -1309,7 +1313,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
                     iob = IOBuffer()
                     name = is_direct_dep ? pkg.name : string(color_string(pkg.name, :light_black))
                     !fancyprint && lock(print_lock) do
-                        isempty(pkg_queue) && printpkgstyle(io, :Precompiling, "environment...")
+                        isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target)
                     end
                     push!(pkg_queue, pkg)
                     started[pkg] = true
@@ -1613,6 +1617,7 @@ function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=
     if compat
         diff && pkgerror("Compat status has no `diff` mode")
         outdated && pkgerror("Compat status has no `outdated` mode")
+
         Operations.print_compat(ctx, pkgs; io)
     else
         Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated)

src/Operations.jl

@@ -142,6 +142,15 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju
         else
             entry.deps = deps_map[pkg.uuid]
         end
+        v = joinpath(source_path(env.project_file, pkg), "Project.toml")
+        if isfile(v)
+            p = Types.read_project(v)
+            entry.gluedeps = p.gluedeps
+            entry.gluepkgs = p.gluepkgs
+            for (name, _) in p.gluedeps
+                delete!(entry.deps, name)
+            end
+        end
         env.manifest[pkg.uuid] = entry
     end
     prune_manifest(env)
@@ -198,32 +207,37 @@ function reset_all_compat!(proj::Project)
     return nothing
 end
 
-function collect_project!(pkg::PackageSpec, path::String,
-                          deps_map::Dict{UUID,Vector{PackageSpec}})
-    deps_map[pkg.uuid] = PackageSpec[]
+function collect_project(pkg::PackageSpec, path::String)
+    deps = PackageSpec[]
+    glues = Set{UUID}()
     project_file = projectfile_path(path; strict=true)
     if project_file === nothing
         pkgerror("could not find project file for package $(err_rep(pkg)) at `$path`")
     end
     project = read_package(project_file)
-    julia_compat = get_compat(project, "julia")
     #=
     # TODO, this should either error or be quiet
+    julia_compat = get_compat(project, "julia")
     if julia_compat !== nothing && !(VERSION in julia_compat)
         println(io, "julia version requirement for package $(err_rep(pkg)) not satisfied")
     end
     =#
     for (name, uuid) in project.deps
         vspec = get_compat(project, name)
-        push!(deps_map[pkg.uuid], PackageSpec(name, uuid, vspec))
+        push!(deps, PackageSpec(name, uuid, vspec))
+    end
+    for (name, uuid) in project.gluedeps
+        vspec = get_compat(project, name)
+        push!(deps, PackageSpec(name, uuid, vspec))
+        push!(glues, uuid)
     end
     if project.version !== nothing
         pkg.version = project.version
     else
         # @warn("project file for $(pkg.name) is missing a `version` entry")
         pkg.version = VersionNumber(0)
     end
-    return
+    return deps, glues
 end
 
 is_tracking_path(pkg) = pkg.path !== nothing
@@ -258,17 +272,22 @@ end
 
 function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String})
     deps_map = Dict{UUID,Vector{PackageSpec}}()
+    glue_map = Dict{UUID,Set{UUID}}()
     if env.pkg !== nothing
         pkg = env.pkg
-        collect_project!(pkg, dirname(env.project_file), deps_map)
+        deps, glues = collect_project(pkg, dirname(env.project_file))
+        deps_map[pkg.uuid] = deps
+        glue_map[pkg.uuid] = glues
         names[pkg.uuid] = pkg.name
     end
     for pkg in pkgs
         path = project_rel_path(env, source_path(env.project_file, pkg))
         if !isdir(path)
             pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`")
         end
-        collect_project!(pkg, path, deps_map)
+        deps, glues = collect_project(pkg, path)
+        deps_map[pkg.uuid] = deps
+        glue_map[pkg.uuid] = glues
     end
 
     fixed = Dict{UUID,Resolve.Fixed}()
@@ -285,7 +304,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU
             idx = findfirst(pkg -> pkg.uuid == uuid, pkgs)
             fix_pkg = pkgs[idx]
         end
-        fixed[uuid] = Resolve.Fixed(fix_pkg.version, q)
+        fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, glue_map[uuid])
     end
     return fixed
 end
@@ -407,6 +426,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
 
     # pkg -> version -> (dependency => compat):
     all_compat = Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}()
+    glue_compat = Dict{UUID,Dict{VersionNumber,Set{UUID}}}()
 
     for (fp, fx) in fixed
         all_compat[fp]   = Dict(fx.version => Dict{UUID,VersionSpec}())
@@ -418,7 +438,8 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
         for uuid in unseen
             push!(seen, uuid)
             uuid in keys(fixed) && continue
-            all_compat_u = get_or_make!(all_compat,   uuid)
+            all_compat_u = get_or_make!(all_compat, uuid)
+            glue_compat_u = get_or_make!(glue_compat, uuid)
 
             uuid_is_stdlib = false
             stdlib_name = ""
@@ -446,35 +467,56 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
                     push!(uuids, other_uuid)
                     all_compat_u_vr[other_uuid] = VersionSpec()
                 end
+
+                if !isempty(proj.gluedeps)
+                    glue_all_compat_u_vr = get_or_make!(glue_compat_u, v)
+                    for (_, other_uuid) in proj.gluedeps
+                        push!(uuids, other_uuid)
+                        all_compat_u_vr[other_uuid] = VersionSpec()
+                        push!(glue_all_compat_u_vr, other_uuid)
+                    end
+                end
             else
                 for reg in registries
                     pkg = get(reg, uuid, nothing)
                     pkg === nothing && continue
                     info = Registry.registry_info(pkg)
-                    for (v, compat_info) in Registry.compat_info(info)
-                        # Filter yanked and if we are in offline mode also downloaded packages
-                        # TODO, pull this into a function
-                        Registry.isyanked(info, v) && continue
-                        if Pkg.OFFLINE_MODE[]
-                            pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v))
-                            is_package_downloaded(env.project_file, pkg_spec) || continue
-                        end
 
-                        # Skip package version that are not the same as external packages in sysimage
-                        if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION
-                            pkgid = Base.PkgId(uuid, pkg.name)
-                            if Base.in_sysimage(pkgid)
-                                pkgorigin = get(Base.pkgorigins, pkgid, nothing)
-                                if pkgorigin !== nothing && pkgorigin.version !== nothing
-                                    if v != pkgorigin.version
-                                        continue
+                    function add_compat!(d, cinfo)
+                        for (v, compat_info) in cinfo
+                            # Filter yanked and if we are in offline mode also downloaded packages
+                            # TODO, pull this into a function
+                            Registry.isyanked(info, v) && continue
+                            if Pkg.OFFLINE_MODE[]
+                                pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v))
+                                is_package_downloaded(env.project_file, pkg_spec) || continue
+                            end
+
+                            # Skip package version that are not the same as external packages in sysimage
+                            if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION
+                                pkgid = Base.PkgId(uuid, pkg.name)
+                                if Base.in_sysimage(pkgid)
+                                    pkgorigin = get(Base.pkgorigins, pkgid, nothing)
+                                    if pkgorigin !== nothing && pkgorigin.version !== nothing
+                                        if v != pkgorigin.version
+                                            continue
+                                        end
                                     end
                                 end
                             end
+                            dv = get_or_make!(d, v)
+                            merge!(dv, compat_info)
+                            union!(uuids, keys(compat_info))
+                        end
+                    end
+                    add_compat!(all_compat_u, Registry.compat_info(info))
+                    glue_compat_info = Registry.glue_compat_info(info)
+                    if glue_compat_info !== nothing
+                        add_compat!(all_compat_u, glue_compat_info)
+                        # Version to Set
+                        for (v, compat_info) in  glue_compat_info
+                            glue_compat_u[v] = keys(compat_info)
                         end
-
-                        all_compat_u[v] = compat_info
-                        union!(uuids, keys(compat_info))
                     end
                 end
             end
@@ -493,7 +535,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance}
         end
     end
 
-    return Resolve.Graph(all_compat, uuid_to_name, reqs, fixed, false, julia_version),
+    return Resolve.Graph(all_compat, glue_compat, uuid_to_name, reqs, fixed, false, julia_version),
            all_compat
 end
 
@@ -1735,9 +1777,13 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String,
     test_project.deps = source_env.project.deps
     # collect test dependencies
     for name in get(source_env.project.targets, target, String[])
-        uuid = get(source_env.project.extras, name, nothing)
+        uuid = nothing
+        for list in [source_env.project.extras, source_env.project.gluedeps]
+            uuid = get(list, name, nothing)
+            uuid === nothing || break
+        end
         if uuid === nothing
-            pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras`")
+            pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras` or `gluedeps`")
         end
         test_project.deps[name] = uuid
     end
@@ -2083,6 +2129,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie
     lpadding = 2
 
     package_statuses = PackageStatusData[]
+    installed_cache = Dict{Base.PkgId, Bool}()
     for (uuid, old, new) in xs
         if Types.is_project_uuid(env, uuid)
             continue
@@ -2116,6 +2163,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie
         no_packages_upgradable &= (!changed || !pkg_upgradable)
         no_visible_packages_heldback &= (!changed || !pkg_heldback)
         no_packages_heldback &= !pkg_heldback
+
         push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed))
     end
 

src/Pkg.jl

@@ -735,9 +735,9 @@ end
 # Precompilation #
 ##################
 
-function _auto_precompile(ctx::Types.Context; warn_loaded = true, already_instantiated = false)
+function _auto_precompile(ctx::Types.Context, pkgs::Vector{String}=String[]; warn_loaded = true, already_instantiated = false)
     if Base.JLOptions().use_compiled_modules == 1 && get_bool_env("JULIA_PKG_PRECOMPILE_AUTO"; default="true")
-        Pkg.precompile(ctx; internal_call=true, warn_loaded = warn_loaded, already_instantiated = already_instantiated)
+        Pkg.precompile(ctx, pkgs; internal_call=true, warn_loaded = warn_loaded, already_instantiated = already_instantiated)
     end
 end