Merge pull request #3090 from JuliaLang/backports-release-1.8

Backports release 1.8

Author: KristofferC

.gitignore

@@ -1,4 +1,5 @@
 /tmp
+/test/tmp
 *.jl.mem
 *.jl.cov
 *.jl.*.cov

CHANGELOG.md

@@ -2,33 +2,53 @@ Pkg v1.8 Release Notes
 ======================
 
 - New `⌃` and `⌅` indicators beside packages in `pkg> status` that have new versions available.
-  `⌅` indicates when new versions cannot be installed.
+  `⌅` indicates when new versions cannot be installed ([#2906]).
 - New `outdated::Bool` kwarg to `Pkg.status` (`--outdated` or `-o` in the REPL mode) to show
-  information about packages not at the latest version.
+  information about packages not at the latest version ([#2284]).
 - New `compat::Bool` kwarg to `Pkg.status` (`--compat` or `-c` in the REPL mode) to show any [compat]
-  entries in the Project.toml.
+  entries in the Project.toml ([#2702]).
 - New `pkg> compat` (and `Pkg.compat`) mode for setting Project compat entries. Provides an interactive editor
   via `pkg> compat`, or direct entry manipulation via `pkg> Foo 0.4,0.5` which can load current entries via tab-completion.
-  i.e. `pkg> compat Fo<TAB>` autocompletes to `pkg> Foo 0.4,0.5` so that the existing entry can be edited.
-- Pkg now only tries to download packages from the package server in case the
-  server tracks a registry that contains the package.
+  i.e. `pkg> compat Fo<TAB>` autocompletes to `pkg> Foo 0.4,0.5` so that the existing entry can be edited ([#2702]).
+- Pkg now only tries to download packages from the package server in case the server tracks a registry that contains
+  the package ([#2689]).
 - `Pkg.instantiate` will now warn when a Project.toml is out of sync with a Manifest.toml. It does this by storing a hash
   of the project deps and compat entries (other fields are ignored) in the manifest when it is resolved, so that any change
-  to the Project.toml deps or compat entries without a re-resolve can be detected.
+  to the Project.toml deps or compat entries without a re-resolve can be detected ([#2815]).
+- If `pkg> add` cannot find a package with the provided name it will now suggest similarly named packages that can be added ([#2985]).
+- The julia version stored in the manifest no longer includes the build number i.e. master will now record as `1.9.0-DEV` ([#2995]).
+- Interrupting a `pkg> test` will now be caught more reliably and exit back to the REPL gracefully ([#2933]).
 
 Pkg v1.7 Release Notes
 ======================
 
 - The format of the `Manifest.toml` file have changed. New manifests will use
-  the new format while old manifest will have their existing format in place.
-  Julia 1.6.2 is compatible with the new format.
+  the new format while old manifest will have their existing format in place ([#2580]).
+  Julia 1.6.2 is compatible with the new format ([#2561]).
 - Registries downloaded from the Pkg Server (not git) are no longer uncompressed into files but instead read directly from the compressed tarball into memory. This improves performance on
-  filesystems which do not handle a large number of files well. To turn this feature off, set the environment variable `JULIA_PKG_UNPACK_REGISTRY=true`.
+  filesystems which do not handle a large number of files well. To turn this feature off, set the environment variable `JULIA_PKG_UNPACK_REGISTRY=true` ([#2431]).
 - It is now possible to use an external `git` executable instead of the default libgit2 library for
-  the downloads that happen via the Git protocol by setting the environment variable `JULIA_PKG_USE_CLI_GIT=true`.
+  the downloads that happen via the Git protocol by setting the environment variable `JULIA_PKG_USE_CLI_GIT=true` ([#2448]).
 - Registries downloaded from the Pkg Server (not git) is now assumed to be immutable. Manual changes to their files might not be picked up by a running Pkg session.
 - The number of packags precompiled in parallel are now limited to 16 unless the
-  environment variable `JULIA_NUM_PRECOMPILE_TASKS` is set.
-- Adding packages by folder name in the REPL mode now requires a prepending a `./` to the folder name package folder is in the current folder, e.g. `add ./Package` is required instead of `add Pacakge`. This is to avoid confusion between the package name `Package` and the local directory `Package`.
-- `rm`, `pin`, and `free` now support the `--all` option, and the api variants gain the `all_pkgs::Bool` kwarg, to perform the operation on all packages within the project or manifest, depending on the mode of the operation.
-- The `mode` keyword for `PackageSpec` has been removed.
+  environment variable `JULIA_NUM_PRECOMPILE_TASKS` is set ([#2552]).
+- Adding packages by folder name in the REPL mode now requires a prepending a `./` to the folder name package folder is in the current folder, e.g. `add ./Package` is required instead of `add Package`. This is to avoid confusion between the package name `Package` and the local directory `Package`.
+- `rm`, `pin`, and `free` now support the `--all` option, and the api variants gain the `all_pkgs::Bool` kwarg, to perform the operation on all packages within the project or manifest, depending on the mode of the operation ([#2432]).
+- The `mode` keyword for `PackageSpec` has been removed ([#2454]).
+
+<!--- Generated by NEWS-update.jl --->
+[#2284]: https://github.com/JuliaLang/Pkg.jl/issues/2284
+[#2431]: https://github.com/JuliaLang/Pkg.jl/issues/2431
+[#2432]: https://github.com/JuliaLang/Pkg.jl/issues/2432
+[#2448]: https://github.com/JuliaLang/Pkg.jl/issues/2448
+[#2454]: https://github.com/JuliaLang/Pkg.jl/issues/2454
+[#2552]: https://github.com/JuliaLang/Pkg.jl/issues/2552
+[#2561]: https://github.com/JuliaLang/Pkg.jl/issues/2561
+[#2580]: https://github.com/JuliaLang/Pkg.jl/issues/2580
+[#2689]: https://github.com/JuliaLang/Pkg.jl/issues/2689
+[#2702]: https://github.com/JuliaLang/Pkg.jl/issues/2702
+[#2815]: https://github.com/JuliaLang/Pkg.jl/issues/2815
+[#2906]: https://github.com/JuliaLang/Pkg.jl/issues/2906
+[#2933]: https://github.com/JuliaLang/Pkg.jl/issues/2933
+[#2985]: https://github.com/JuliaLang/Pkg.jl/issues/2985
+[#2995]: https://github.com/JuliaLang/Pkg.jl/issues/2995

docs/NEWS-update.jl

@@ -0,0 +1,19 @@
+# Script to automatically insert Markdown footnotes for all [#xxxx] issue
+# cross-references in the CHANGELOG file.
+
+NEWS = get(ARGS, 1, "CHANGELOG.md")
+
+s = read(NEWS, String)
+
+m = match(r"\[#[0-9]+\]:", s)
+if m !== nothing
+    s = s[1:m.offset-1]
+end
+
+footnote(n) = "[#$n]: https://github.com/JuliaLang/Pkg.jl/issues/$n"
+N = map(m -> parse(Int,m.captures[1]), eachmatch(r"\[#([0-9]+)\]", s))
+foots = join(map(footnote, sort!(unique(N))), "\n")
+
+open(NEWS, "w") do f
+    println(f, s, "<!--- Generated by NEWS-update.jl --->\n", foots)
+end

docs/src/creating-packages.md

@@ -309,5 +309,5 @@ To support the various usecases in the Julia package ecosystem, the Pkg develope
 * [`Preferences.jl`](https://github.com/JuliaPackaging/Preferences.jl) allows packages to read and write preferences to the top-level `Project.toml`.
   These preferences can be read at runtime or compile-time, to enable or disable different aspects of package behavior.
   Packages previously would write out files to their own package directories to record options set by the user or environment, but this is highly discouraged now that `Preferences` is available.
-  Preferences are available from Juilia 1.6 onward.
+  Preferences are available from Julia 1.6 onward.
 

src/API.jl

@@ -1813,7 +1813,7 @@ function compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; io
             resolve(ctx)
         catch e
             if e isa ResolverError
-                printpkgstyle(io, :Error, e.msg, color = Base.warn_color())
+                printpkgstyle(io, :Error, string(e.msg), color = Base.warn_color())
             else
                 rethrow()
             end

src/Operations.jl

@@ -116,6 +116,9 @@ function is_instantiated(env::EnvCache; platform = HostPlatform())::Bool
         if idx === nothing
             push!(pkgs, Types.PackageSpec(name=env.pkg.name, uuid=env.pkg.uuid, version=env.pkg.version, path=dirname(env.project_file)))
         end
+    else
+        # Make sure artifacts for project exist even if it is not a package
+        check_artifacts_downloaded(dirname(env.project_file); platform) || return false
     end
     # Make sure all paths/artifacts exist
     return all(pkg -> is_package_downloaded(env.project_file, pkg; platform), pkgs)
@@ -287,22 +290,24 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU
     return fixed
 end
 
+# drops build detail in version but keeps the main prerelease context
+# i.e. dropbuild(v"2.0.1-rc1.21321") == v"2.0.1-rc1"
+dropbuild(v::VersionNumber) = VersionNumber(v.major, v.minor, v.patch, isempty(v.prerelease) ? () : (v.prerelease[1],))
 
 # Resolve a set of versions given package version specs
 # looks at uuid, version, repo/path,
 # sets version to a VersionNumber
 # adds any other packages which may be in the dependency graph
-# all versioned packges should have a `tree_hash`
+# all versioned packages should have a `tree_hash`
 function resolve_versions!(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}, julia_version)
     # compatibility
     if julia_version !== nothing
-        env.manifest.julia_version = julia_version
+        # only set the manifest julia_version if ctx.julia_version is not nothing
+        env.manifest.julia_version = dropbuild(VERSION)
         v = intersect(julia_version, get_compat(env.project, "julia"))
         if isempty(v)
             @warn "julia version requirement for project not satisfied" _module=nothing _file=nothing
         end
-    else
-        env.manifest.julia_version = VERSION
     end
     names = Dict{UUID, String}(uuid => name for (uuid, (name, version)) in stdlibs())
     # recursive search for packages which are tracking a path
@@ -632,8 +637,7 @@ function download_artifacts(env::EnvCache;
         pkg_root = source_path(env.project_file, pkg, julia_version)
         pkg_root === nothing || push!(pkg_roots, pkg_root)
     end
-    envpkg = env.pkg
-    envpkg === nothing || push!(pkg_roots, envpkg.path)
+    push!(pkg_roots, dirname(env.project_file))
     for pkg_root in pkg_roots
         for (artifacts_toml, artifacts) in collect_artifacts(pkg_root; platform)
             # For each Artifacts.toml, install each artifact we've collected from it
@@ -976,12 +980,12 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false)
         local build_project_override, build_project_preferences
         if isfile(projectfile_path(builddir(source_path)))
             build_project_override = nothing
-            with_load_path([builddir(source_path)]) do
+            with_load_path([builddir(source_path), Base.LOAD_PATH...]) do
                 build_project_preferences = Base.get_preferences()
             end
         else
             build_project_override = gen_target_project(ctx, pkg, source_path, "build")
-            with_load_path([projectfile_path(source_path)]) do
+            with_load_path([projectfile_path(source_path), Base.LOAD_PATH...]) do
                 build_project_preferences = Base.get_preferences()
             end
         end
@@ -1708,12 +1712,12 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
         local test_project_preferences, test_project_override
         if isfile(projectfile_path(testdir(source_path)))
             test_project_override = nothing
-            with_load_path(testdir(source_path)) do
+            with_load_path([testdir(source_path), Base.LOAD_PATH...]) do
                 test_project_preferences = Base.get_preferences()
             end
         else
             test_project_override = gen_target_project(ctx, pkg, source_path, "test")
-            with_load_path(projectfile_path(source_path)) do
+            with_load_path([projectfile_path(source_path), Base.LOAD_PATH...]) do
                 test_project_preferences = Base.get_preferences()
             end
         end
@@ -1727,7 +1731,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
             printpkgstyle(ctx.io, :Testing, "Running tests...")
             flush(ctx.io)
             cmd = gen_test_code(testfile(source_path); coverage=coverage, julia_args=julia_args, test_args=test_args)
-            p = run(pipeline(ignorestatus(cmd), stdin = stdin, stdout = sandbox_ctx.io, stderr = stderr_f()), wait = false)
+            p = run(pipeline(ignorestatus(cmd), stdout = sandbox_ctx.io, stderr = stderr_f()), wait = false)
             interrupted = false
             try
                 wait(p)

src/REPLMode/REPLMode.jl

@@ -717,13 +717,21 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol})
         API.add(string.(available_pkgs))
     elseif lower_resp in ["o"]
         editable_envs = filter(v -> v != "@stdlib", LOAD_PATH)
-        expanded_envs = Base.load_path_expand.(editable_envs)
-        envs = convert(Vector{String}, filter(x -> !isnothing(x), expanded_envs))
         option_list = String[]
         keybindings = Char[]
-        for i in 1:length(envs)
-            push!(option_list, "$(i): $(pathrepr(envs[i])) ($(editable_envs[i]))")
-            push!(keybindings, only("$i"))
+        shown_envs = String[]
+        # We use digits 1-9 as keybindings in the env selection menu
+        # That's why we can display at most 9 items in the menu
+        for i in 1:min(length(editable_envs), 9)
+            env = editable_envs[i]
+            expanded_env = Base.load_path_expand(env)
+
+            isnothing(expanded_env) && continue
+
+            n = length(option_list) + 1
+            push!(option_list, "$(n): $(pathrepr(expanded_env)) ($(env))")
+            push!(keybindings, only("$n"))
+            push!(shown_envs, expanded_env)
         end
         menu = TerminalMenus.RadioMenu(option_list, keybindings=keybindings, pagesize=length(option_list))
         print(ctx.io, "\e[1A\e[1G\e[0J") # go up one line, to the start, and clear it
@@ -738,7 +746,7 @@ function try_prompt_pkg_add(pkgs::Vector{Symbol})
             rethrow()
         end
         choice == -1 && return false
-        API.activate(envs[choice]) do
+        API.activate(shown_envs[choice]) do
             API.add(string.(available_pkgs))
         end
     elseif (lower_resp in ["n"])

src/utils.jl

@@ -58,6 +58,7 @@ set_readonly(::Nothing) = nothing
 
 # try to call realpath on as much as possible
 function safe_realpath(path)
+    isempty(path) && return path
     if ispath(path)
         try
             return realpath(path)

test/artifacts.jl

@@ -488,7 +488,7 @@ end
             "AugmentedPlatform",
             "flooblecrank" => "disengaged",
         )
-    
+
         p = HostPlatform()
         p["flooblecrank"] = "engaged"
         add_this_pkg(; platform=p)
@@ -770,4 +770,16 @@ end
     end
 end
 
+@testset "artifacts for non package project" begin
+    temp_pkg_dir() do tmpdir
+        artifacts_toml = joinpath(tmpdir, "Artifacts.toml")
+        cp(joinpath(@__DIR__, "test_packages", "ArtifactInstallation", "Artifacts.toml"), artifacts_toml)
+        Pkg.activate(tmpdir)
+        cts_hash = artifact_hash("collapse_the_symlink", artifacts_toml)
+        @test !artifact_exists(cts_hash)
+        Pkg.instantiate()
+        @test artifact_exists(cts_hash)
+    end
+end
+
 end # module

test/manifest/formats/v2.0/Manifest.toml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.7.0-DEV.1199"
+julia_version = "1.7.0-DEV"
 manifest_format = "2.0"
 some_other_field = "other"
 some_other_data = [1,2,3,4]