From 85155e12997a273a21b7926af4a81193a3ef6c15 Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Sat, 24 Sep 2022 22:32:19 +0800 Subject: [PATCH 01/41] Set `intersection = 1` during `intersect_sub_datatype` (#46882) (cherry picked from commit e6d99792e6dd0cffe41e98b8d642cf07e64364c5) --- src/subtype.c | 2 +- test/subtype.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/subtype.c b/src/subtype.c index 1213cd53af2f6..bb229d4b76bd0 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -2840,7 +2840,7 @@ static jl_value_t *intersect_sub_datatype(jl_datatype_t *xd, jl_datatype_t *yd, JL_GC_PUSHARGS(env, envsz); jl_stenv_t tempe; init_stenv(&tempe, env, envsz); - tempe.ignore_free = 1; + tempe.intersection = tempe.ignore_free = 1; if (subtype_in_env(isuper, super_pattern, &tempe)) { jl_value_t *wr = wrapper; int i; diff --git a/test/subtype.jl b/test/subtype.jl index 5bb2037d7852b..9545838bff141 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2031,3 +2031,10 @@ S46735{B<:Val, M<:AbstractMatrix} = Tuple{<:Union{B, <:Val{<:B}},M,<:(Union{Abst A46735{B<:Val, M<:AbstractMatrix} = Tuple{<:Union{B, <:Val{<:B}},M,Union{AbstractMatrix{B}, AbstractMatrix{<:Vector{<:B}}}} @testintersect(A46735{B} where {B}, A46735, !Union{}) @testintersect(A46735{B, M} where {B, M}, A46735, !Union{}) + +#issue #46871 +struct A46871{T, N, M} <: AbstractArray{T, N} end +struct B46871{T, N} <: Ref{A46871{T, N, N}} end +for T in (B46871{Int, N} where {N}, B46871{Int}) # intentional duplication + @testintersect(T, Ref{<:AbstractArray{<:Real, 3}}, B46871{Int, 3}) +end From 1b70b0f39bb62650948aa4ef49139d6d71345913 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Sun, 25 Sep 2022 15:55:10 +0600 Subject: [PATCH 02/41] Don't prompt field names for tuples at the REPL (#46897) (cherry picked from commit d5cde865f24d2c3e7041aee8ea464eb6b6045a2a) --- stdlib/REPL/src/REPLCompletions.jl | 2 +- stdlib/REPL/test/replcompletions.jl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 07b904c6abb84..170ce53061a3a 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -195,7 +195,7 @@ function complete_symbol(sym::String, @nospecialize(ffunc), context_module::Modu t = typeof(t.parameters[1]) end # Only look for fields if this is a concrete type - if isconcretetype(t) + if isconcretetype(t) && !(t <: Tuple) fields = fieldnames(t) for field in fields s = string(field) diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 1d59b6e057882..3d88b326d94a4 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -270,6 +270,12 @@ let @test isempty(c) end +# issue 46800: (3,2). errors in the REPL +let + c, r = test_complete("(3,2).") + @test isempty(c) +end + # inexistent completion inside a string let s = "Base.print(\"lol" c, r, res = test_complete(s) From 1dcd60fc07b1650d6a1db24a57d2d91a8a86d1d4 Mon Sep 17 00:00:00 2001 From: William Moses Date: Mon, 26 Sep 2022 17:35:04 -0400 Subject: [PATCH 03/41] Don't crash on variable sized gc allocations (#46914) (cherry picked from commit 8003563a5041f4253443b589ab72bda3817cdb8a) --- src/llvm-alloc-opt.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index 3f270cde8d96d..2347b51c899c9 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -314,7 +314,10 @@ ssize_t Optimizer::getGCAllocSize(Instruction *I) if (call->getCalledOperand() != pass.alloc_obj_func) return -1; assert(call->arg_size() == 3); - size_t sz = (size_t)cast(call->getArgOperand(1))->getZExtValue(); + auto CI = dyn_cast(call->getArgOperand(1)); + if (!CI) + return -1; + size_t sz = (size_t)CI->getZExtValue(); if (sz < IntegerType::MAX_INT_BITS / 8 && sz < INT32_MAX) return sz; return -1; From d9fe0505a5e3e33425416c2a0fa0952936624e52 Mon Sep 17 00:00:00 2001 From: Hendrik Ranocha Date: Tue, 27 Sep 2022 08:50:50 +0200 Subject: [PATCH 04/41] improve type stability of `handle_message(logger, ...` (#46906) (cherry picked from commit 6c0aa6dcbe10151501e4a1756876ea330734ab12) --- base/logging.jl | 2 +- stdlib/Logging/src/ConsoleLogger.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/logging.jl b/base/logging.jl index db7ddf37c676d..fe40c482c960f 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -668,7 +668,7 @@ function handle_message(logger::SimpleLogger, level::LogLevel, message, _module, remaining > 0 || return end buf = IOBuffer() - stream = logger.stream + stream::IO = logger.stream if !(isopen(stream)::Bool) stream = stderr end diff --git a/stdlib/Logging/src/ConsoleLogger.jl b/stdlib/Logging/src/ConsoleLogger.jl index 3d283e7a33456..66e1c935b41fd 100644 --- a/stdlib/Logging/src/ConsoleLogger.jl +++ b/stdlib/Logging/src/ConsoleLogger.jl @@ -117,7 +117,7 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module # Generate a text representation of the message and all key value pairs, # split into lines. msglines = [(indent=0, msg=l) for l in split(chomp(convert(String, string(message))::String), '\n')] - stream = logger.stream + stream::IO = logger.stream if !(isopen(stream)::Bool) stream = stderr end From 2666073e8a33edbd919d7b7eea8689683751325c Mon Sep 17 00:00:00 2001 From: Perry Fraser Date: Tue, 27 Sep 2022 13:24:46 -0400 Subject: [PATCH 05/41] Don't error when transposing an empty buffer (#46925) (cherry picked from commit 757f21e0bdd6827bc8a283dc290069ebe7c87912) --- stdlib/REPL/src/LineEdit.jl | 2 +- stdlib/REPL/test/lineedit.jl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 3854a81ee0bcb..9215c9f574879 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -1086,7 +1086,7 @@ end function edit_transpose_chars(buf::IOBuffer) # Moving left but not transpoing anything is intentional, and matches Emacs's behavior - eof(buf) && char_move_left(buf) + eof(buf) && position(buf) !== 0 && char_move_left(buf) position(buf) == 0 && return false char_move_left(buf) pos = position(buf) diff --git a/stdlib/REPL/test/lineedit.jl b/stdlib/REPL/test/lineedit.jl index 87028e239d5b8..9b0569bb08424 100644 --- a/stdlib/REPL/test/lineedit.jl +++ b/stdlib/REPL/test/lineedit.jl @@ -376,6 +376,8 @@ let buf = IOBuffer() LineEdit.edit_transpose_chars(buf) @test content(buf) == "βγαδε" + + # Transposing a one-char buffer should behave like Emacs seek(buf, 0) @inferred(LineEdit.edit_clear(buf)) edit_insert(buf, "a") @@ -385,6 +387,13 @@ let buf = IOBuffer() LineEdit.edit_transpose_chars(buf) @test content(buf) == "a" @test position(buf) == 0 + + # Transposing an empty buffer shouldn't implode + seek(buf, 0) + LineEdit.edit_clear(buf) + LineEdit.edit_transpose_chars(buf) + @test content(buf) == "" + @test position(buf) == 0 end @testset "edit_word_transpose" begin From 5d30157185340fd10ff449c2cb595d6cfb553ead Mon Sep 17 00:00:00 2001 From: Fredrik Ekre Date: Tue, 27 Sep 2022 21:15:51 +0200 Subject: [PATCH 06/41] Fix shift direction of circshift! for vectors (#46759) This patch fixes the shifting direction for circshift!(x::AbstractVector, n::Integer), which was opposite the direction of circshift(x, n) and circshift!(y, x, n). In addition, this patch fixes the method to also work correctly with offset arrays. Fixes #46533. (cherry picked from commit f1c4d54696c2d296a10a8f8d659f6a3e85da9a29) --- base/abstractarray.jl | 5 +++-- test/arrayops.jl | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index a27a9cd7271b5..864c0cc92dc5f 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3187,8 +3187,9 @@ function circshift!(a::AbstractVector, shift::Integer) n == 0 && return shift = mod(shift, n) shift == 0 && return - reverse!(a, 1, shift) - reverse!(a, shift+1, length(a)) + l = lastindex(a) + reverse!(a, firstindex(a), l-shift) + reverse!(a, l-shift+1, lastindex(a)) reverse!(a) return a end diff --git a/test/arrayops.jl b/test/arrayops.jl index 22d38fddd3636..7587b176a0deb 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -761,6 +761,18 @@ end @test circshift(src, 1) == src src = zeros(Bool, (4,0)) @test circshift(src, 1) == src + + # 1d circshift! (https://github.com/JuliaLang/julia/issues/46533) + a = [1:5;] + @test circshift!(a, 1) === a + @test a == circshift([1:5;], 1) == [5, 1, 2, 3, 4] + a = [1:5;] + @test circshift!(a, -2) === a + @test a == circshift([1:5;], -2) == [3, 4, 5, 1, 2] + a = [1:5;] + oa = OffsetVector(copy(a), -1) + @test circshift!(oa, 1) === oa + @test oa == circshift(OffsetVector(a, -1), 1) end @testset "circcopy" begin From 752cc699a8f553a1db6d94316a5237055faa7ba4 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Fri, 30 Sep 2022 15:37:01 +0200 Subject: [PATCH 07/41] bump SparseArrays to latest release-1.8 --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/SparseArrays.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/md5 create mode 100644 deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/sha512 delete mode 100644 deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/md5 delete mode 100644 deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/sha512 diff --git a/deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/md5 b/deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/md5 new file mode 100644 index 0000000000000..1e22b80501a93 --- /dev/null +++ b/deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/md5 @@ -0,0 +1 @@ +e62b7c98591daeddeefc775c67a5a174 diff --git a/deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/sha512 b/deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/sha512 new file mode 100644 index 0000000000000..8aff3bdef2ba0 --- /dev/null +++ b/deps/checksums/SparseArrays-1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6.tar.gz/sha512 @@ -0,0 +1 @@ +10552c210f4152611e65aa7a6f92e382ab3aea51abfd712ca8eb1f42826b1f0c9358bb98a0e54a8b4167f9cb9f9a84a98020be4922084480a3faeae76c072eaf diff --git a/deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/md5 b/deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/md5 deleted file mode 100644 index dc29e4d73a572..0000000000000 --- a/deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c800768d427797e16c1dfb050e3615f6 diff --git a/deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/sha512 b/deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/sha512 deleted file mode 100644 index 093da363d2c6a..0000000000000 --- a/deps/checksums/SparseArrays-aa51c9b82d952502139213715c9b077ec36c4623.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -fe7966ba74a473c99c591bcc18ac8827949b8a764bf4e70991567cb0eb7fc8adeb2674ee09467ef431684d4ad3dbfc81d5f3df0bea7c48ca3a212b8de446303a diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index 1d93c36f6d7f0..cdee6b6ffc516 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = aa51c9b82d952502139213715c9b077ec36c4623 +SPARSEARRAYS_SHA1 = 1d3ec8656939cc8a2c6c67fa51b76995fd72b8a6 SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 From ddbb5bc3dd8871d62b200a65b98c2e1523b0cecd Mon Sep 17 00:00:00 2001 From: woclass Date: Sat, 8 Oct 2022 23:43:26 +0800 Subject: [PATCH 08/41] [win] Backport win MSYS2 build pr (#46988) * Build/win: Build with MSYS2 (#46140) * Makefile: MSYS2: close path conversion for `DEP_LIBS` Automatic path conversion will replace `:` with `;` * Makefile: MSYS2: use `cygpath` for path convert ref: https://www.msys2.org/docs/filesystem-paths/#manual-unix-windows-path-conversion * devdoc/win/msys2: add build steps * devdoc/win/msys2: Add x86/x64 build notes * devdoc/win/msys2: apply sugestions Co-Authored-By: Elliot Saba * Instead of `CC_WITH_ENV`, scope environment variables to targets * Fix whitespace Co-authored-by: Elliot Saba * Disable MSYS2's path munging for `stringreplace` (#46803) This was causing our rewriting of the loader's RPATH emulation to fail after running `make install`, as MSYS2 was rewriting our list that looks like: ``` ../bin/libgcc_s_seh-1.dll:../bin/libopenlibm.dll:@../bin/libjulia-internal.dll:@../bin/libjulia-codegen.dll: ``` Into one that looked like: ``` ..\bin\libgcc_s_seh-1.dll;..\bin\libopenlibm.dll;@..\bin\libjulia-internal.dll;@..\bin\libjulia-codegen.dll; ``` By exporting `MSYS2_ARG_CONV_EXCL='*'` for all `stringreplace` invocations, we dodge this issue, as documented by MSYS2 [0]. [0] https://www.msys2.org/wiki/Porting/#filesystem-namespaces * [win] Disable MSYS2 path munging when calling `is.exe` (#46867) Tragically, I believe MSYS2 is messing with options such as `/VERYSILENT` turning them instead into `C:\msys2\VERYSILENT` or similar. Co-authored-by: Elliot Saba --- Make.inc | 6 +-- Makefile | 11 ++++-- cli/Makefile | 8 ++++ doc/src/devdocs/build/windows.md | 66 +++++++++++++++++++++++++++++--- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/Make.inc b/Make.inc index e1420dd5bac74..dbf723f7ec4ee 100644 --- a/Make.inc +++ b/Make.inc @@ -1391,13 +1391,13 @@ define symlink_target # (from, to-dir, to-name) CLEAN_TARGETS += clean-$$(abspath $(2)/$(3)) clean-$$(abspath $(2)/$(3)): ifeq ($(BUILD_OS), WINNT) - -cmd //C rmdir $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) + -cmd //C rmdir $$(call cygpath_w,$(2)/$(3)) else -rm -r $$(abspath $(2)/$(3)) endif $$(abspath $(2)/$(3)): | $$(abspath $(2)) ifeq ($$(BUILD_OS), WINNT) - @cmd //C mklink //J $$(call mingw_to_dos,$(2)/$(3),cd $(2) &&) $$(call mingw_to_dos,$(1),) + @cmd //C mklink //J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1)) else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS))) @cmd /C mklink /J $$(call cygpath_w,$(2)/$(3)) $$(call cygpath_w,$(1)) else ifdef JULIA_VAGRANT_BUILD @@ -1415,7 +1415,7 @@ WINE ?= wine # many of the following targets must be = not := because the expansion of the makefile functions (and $1) shouldn't happen until later ifeq ($(BUILD_OS), WINNT) # MSYS spawn = $(1) -cygpath_w = $(1) +cygpath_w = `cygpath -w $(1)` else ifneq (,$(findstring CYGWIN,$(BUILD_OS))) # Cygwin spawn = $(1) cygpath_w = `cygpath -w $(1)` diff --git a/Makefile b/Makefile index d38311dce720a..57b5953107914 100644 --- a/Makefile +++ b/Makefile @@ -222,8 +222,10 @@ endif endif endif +# Note that we disable MSYS2's path munging here, as otherwise +# it replaces our `:`-separated list as a `;`-separated one. define stringreplace - $(build_depsbindir)/stringreplace $$(strings -t x - $1 | grep $2 | awk '{print $$1;}') $3 255 "$(call cygpath_w,$1)" + MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - $1 | grep $2 | awk '{print $$1;}') $3 255 "$(call cygpath_w,$1)" endef @@ -446,8 +448,9 @@ endif exe: - # run Inno Setup to compile installer - $(call spawn,$(JULIAHOME)/dist-extras/inno/iscc.exe /DAppVersion=$(JULIA_VERSION) /DSourceDir="$(call cygpath_w,$(BUILDROOT)/julia-$(JULIA_COMMIT))" /DRepoDir="$(call cygpath_w,$(JULIAHOME))" /F"$(JULIA_BINARYDIST_FILENAME)" /O"$(call cygpath_w,$(BUILDROOT))" $(INNO_ARGS) $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.iss)) + # run Inno Setup to compile installer. + # Note that we disable MSYS2 path munging, as it interferes with the `/` options: + MSYS2_ARG_CONV_EXCL='*' $(call spawn,$(JULIAHOME)/dist-extras/inno/iscc.exe /DAppVersion=$(JULIA_VERSION) /DSourceDir="$(call cygpath_w,$(BUILDROOT)/julia-$(JULIA_COMMIT))" /DRepoDir="$(call cygpath_w,$(JULIAHOME))" /F"$(JULIA_BINARYDIST_FILENAME)" /O"$(call cygpath_w,$(BUILDROOT))" $(INNO_ARGS) $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.iss)) chmod a+x "$(BUILDROOT)/$(JULIA_BINARYDIST_FILENAME).exe" app: @@ -572,7 +575,7 @@ win-extras: cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) https://www.jrsoftware.org/download.php/is.exe && \ chmod a+x is.exe && \ - $(call spawn, $(JULIAHOME)/dist-extras/is.exe /DIR="$(call cygpath_w,$(JULIAHOME)/dist-extras/inno)" /PORTABLE=1 /CURRENTUSER /VERYSILENT) + MSYS2_ARG_CONV_EXCL='*' $(call spawn, $(JULIAHOME)/dist-extras/is.exe /DIR="$(call cygpath_w,$(JULIAHOME)/dist-extras/inno)" /PORTABLE=1 /CURRENTUSER /VERYSILENT) # various statistics about the build that may interest the user ifeq ($(USE_SYSTEM_LLVM), 1) diff --git a/cli/Makefile b/cli/Makefile index 11855ee6244dc..16b479954c724 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -29,6 +29,14 @@ endif # Build list of dependent libraries that must be opened SHIPFLAGS += -DDEP_LIBS="\"$(LOADER_BUILD_DEP_LIBS)\"" DEBUGFLAGS += -DDEP_LIBS="\"$(LOADER_DEBUG_BUILD_DEP_LIBS)\"" +ifneq (,$(findstring MINGW,$(shell uname))) +# In MSYS2, do not perform path conversion for `DEP_LIBS`. +# https://www.msys2.org/wiki/Porting/#filesystem-namespaces +# We define this environment variable for only these two object files, +# as they're the only ones that require it at the time of writing. +$(BUILDDIR)/loader_lib.o: export MSYS2_ARG_CONV_EXCL = -DDEP_LIBS= +$(BUILDDIR)/loader_lib.dbg.obj: export MSYS2_ARG_CONV_EXCL = -DDEP_LIBS= +endif # MSYS2 EXE_OBJS := $(BUILDDIR)/loader_exe.o EXE_DOBJS := $(BUILDDIR)/loader_exe.dbg.obj diff --git a/doc/src/devdocs/build/windows.md b/doc/src/devdocs/build/windows.md index fef4413db7d1a..1f5f03cb7aac8 100644 --- a/doc/src/devdocs/build/windows.md +++ b/doc/src/devdocs/build/windows.md @@ -118,11 +118,67 @@ MinGW-w64 compilers available through Cygwin's package manager. ### Compiling with MinGW/MSYS2 -Compiling Julia from source using [MSYS2](https://msys2.github.io) has worked in the past -but is not actively supported. Pull requests to restore support would be welcome. See a -[past version of this -file](https://github.com/JuliaLang/julia/blob/v0.6.0/README.windows.md) for the former -instructions for compiling using MSYS2. +> MSYS2 provides a robust MSYS experience. + +Note: MSYS2 requires **64 bit** Windows 7 or newer. + + 1. Install and configure [MSYS2](https://www.msys2.org/), Software Distribution + and Building Platform for Windows. + + 1. Download and run the latest installer for the + [64-bit](https://github.com/msys2/msys2-installer/releases/latest) distribution. + The installer will have a name like `msys2-x86_64-yyyymmdd.exe`. + + 2. Open MSYS2. Update package database and base packages: + ```sh + pacman -Syu + ``` + + 3. Exit and restart MSYS2, Update the rest of the base packages: + ```sh + pacman -Syu + ``` + + 3. Then install tools required to build julia: + ```sh + # tools + pacman -S cmake diffutils git m4 make patch tar p7zip curl python + + # For 64 bit Julia, install x86_64 + pacman -S mingw-w64-x86_64-gcc + # For 32 bit Julia, install i686 + pacman -S mingw-w64-i686-gcc + ``` + + 4. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. + + + 2. Build Julia and its dependencies with pre-build dependencies. + + 1. Open a new [**MINGW64/MINGW32 shell**](https://www.msys2.org/docs/environments/#overview). + Currently we can't use both mingw32 and mingw64, + so if you want to build the x86_64 and i686 versions, + you'll need to build them in each environment separately. + + 2. and clone the Julia sources + ```sh + git clone https://github.com/JuliaLang/julia.git + cd julia + ``` + + 3. Start the build + ```sh + make -j$(nproc) + ``` + + > Protip: build in dir + > ```sh + > make O=julia-mingw-w64 configure + > echo 'ifeq ($(BUILDROOT),$(JULIAHOME)) + > $(error "in-tree build disabled") + > endif' >> Make.user + > make -C julia-mingw-w64 + > ``` ### Cross-compiling from Unix (Linux/Mac/WSL) From 6fe0031f12cd4d66d65e0927bd1be42267639df4 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 17 Aug 2022 10:30:54 +0200 Subject: [PATCH 09/41] Fix serialization of code instances (#46373) --- src/dump.c | 14 ++++++++------ test/precompile.jl | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/dump.c b/src/dump.c index 8cb2f901ee45b..dfd80f2a5fc63 100644 --- a/src/dump.c +++ b/src/dump.c @@ -583,13 +583,15 @@ static int jl_serialize_generic(jl_serializer_state *s, jl_value_t *v) JL_GC_DIS return 0; } -static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_t *codeinst, int skip_partial_opaque, int internal) JL_GC_DISABLED +static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_t *codeinst, + int skip_partial_opaque, int internal, + int force) JL_GC_DISABLED { if (internal > 2) { while (codeinst && !codeinst->relocatability) codeinst = codeinst->next; } - if (jl_serialize_generic(s, (jl_value_t*)codeinst)) { + if (!force && jl_serialize_generic(s, (jl_value_t*)codeinst)) { return; } @@ -609,7 +611,7 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ if (write_ret_type && codeinst->rettype_const && jl_typeis(codeinst->rettype_const, jl_partial_opaque_type)) { if (skip_partial_opaque) { - jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, internal); + jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, internal, 0); return; } else { @@ -636,7 +638,7 @@ static void jl_serialize_code_instance(jl_serializer_state *s, jl_code_instance_ jl_serialize_value(s, jl_nothing); } write_uint8(s->s, codeinst->relocatability); - jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, internal); + jl_serialize_code_instance(s, codeinst->next, skip_partial_opaque, internal, 0); } enum METHOD_SERIALIZATION_MODE { @@ -893,10 +895,10 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li } jl_serialize_value(s, (jl_value_t*)backedges); jl_serialize_value(s, (jl_value_t*)NULL); //callbacks - jl_serialize_code_instance(s, mi->cache, 1, internal); + jl_serialize_code_instance(s, mi->cache, 1, internal, 0); } else if (jl_is_code_instance(v)) { - jl_serialize_code_instance(s, (jl_code_instance_t*)v, 0, 2); + jl_serialize_code_instance(s, (jl_code_instance_t*)v, 0, 2, 1); } else if (jl_typeis(v, jl_module_type)) { jl_serialize_module(s, (jl_module_t*)v); diff --git a/test/precompile.jl b/test/precompile.jl index a8937581cb5d0..072d0ba694699 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1290,3 +1290,20 @@ precompile_test_harness("__init__ cachepath") do load_path """) @test isa((@eval (using InitCachePath; InitCachePath)), Module) end + +precompile_test_harness("issue #46296") do load_path + write(joinpath(load_path, "CodeInstancePrecompile.jl"), + """ + module CodeInstancePrecompile + + mi = first(methods(identity)).specializations[1] + ci = Core.CodeInstance(mi, Any, nothing, nothing, zero(Int32), typemin(UInt), + typemax(UInt), zero(UInt32), zero(UInt32), nothing, 0x00) + + __init__() = @assert ci isa Core.CodeInstance + + end + """) + Base.compilecache(Base.PkgId("CodeInstancePrecompile")) + (@eval (using CodeInstancePrecompile)) +end From 14dcba20b8ace059f89fb068e8faf7111b3899ec Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 17 Jun 2022 03:41:29 -0500 Subject: [PATCH 10/41] `Base.stale_cachefile`: allow `ftime_req` to be greater than `ftime` by up to one microsecond (to compensate for Windows tar) (#45552) (cherry picked from commit 4c39647aed7d2a36e38bd6c8c04469e22bbf5c33) --- base/loading.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 8625db41df0a2..bbb111c955ecf 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2144,11 +2144,13 @@ end end for chi in includes f, ftime_req = chi.filename, chi.mtime - # Issue #13606: compensate for Docker images rounding mtimes - # Issue #20837: compensate for GlusterFS truncating mtimes to microseconds - # The `ftime != 1.0` condition below provides compatibility with Nix mtime. ftime = mtime(f) - if ftime != ftime_req && ftime != floor(ftime_req) && ftime != trunc(ftime_req, digits=6) && ftime != 1.0 + is_stale = ( ftime != ftime_req ) && + ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes + ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds + ( ftime != 1.0 ) && # PR #43090: provide compatibility with Nix mtime. + !( 0 < (ftime_req - ftime) < 1e-6 ) # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond + if is_stale @debug "Rejecting stale cache file $cachefile (mtime $ftime_req) because file $f (mtime $ftime) has changed" return true end From eb4cf73716434867d52d299d34d97305faee2ee3 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 21 Jun 2022 23:25:30 -0400 Subject: [PATCH 11/41] Take into account color and unicode in matrix alignment (#45751) Without this, alignment would count characters rather than textwidth as well as counting inline escape sequences in colored output. Fix that by using uncolored printing for alignment and textwidth rather than number of codepoints. (cherry picked from commit 626acd40562644285649427118be5b213a9b7b90) --- base/show.jl | 38 ++++++++++++++++++++++---------------- test/show.jl | 13 +++++++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/base/show.jl b/base/show.jl index 381a17f188482..2a32a59fef7c4 100644 --- a/base/show.jl +++ b/base/show.jl @@ -2715,6 +2715,9 @@ function dump(arg; maxdepth=DUMP_DEFAULT_MAXDEPTH) dump(IOContext(stdout, :limit => true, :module => mod), arg; maxdepth=maxdepth) end +nocolor(io::IO) = IOContext(io, :color => false) +alignment_from_show(io::IO, x::Any) = + textwidth(sprint(show, x, context=nocolor(io), sizehint=0)) """ `alignment(io, X)` returns a tuple (left,right) showing how many characters are @@ -2732,35 +2735,38 @@ julia> Base.alignment(stdout, 1 + 10im) (3, 5) ``` """ -alignment(io::IO, x::Any) = (0, length(sprint(show, x, context=io, sizehint=0))) -alignment(io::IO, x::Number) = (length(sprint(show, x, context=io, sizehint=0)), 0) -alignment(io::IO, x::Integer) = (length(sprint(show, x, context=io, sizehint=0)), 0) +alignment(io::IO, x::Any) = (0, alignment_from_show(io, x)) +alignment(io::IO, x::Number) = (alignment_from_show(io, x), 0) +alignment(io::IO, x::Integer) = (alignment_from_show(io, x), 0) function alignment(io::IO, x::Real) - m = match(r"^(.*?)((?:[\.eEfF].*)?)$", sprint(show, x, context=io, sizehint=0)) - m === nothing ? (length(sprint(show, x, context=io, sizehint=0)), 0) : - (length(m.captures[1]), length(m.captures[2])) + s = sprint(show, x, context=nocolor(io), sizehint=0) + m = match(r"^(.*?)((?:[\.eEfF].*)?)$", s) + m === nothing ? (textwidth(s), 0) : + (textwidth(m.captures[1]), textwidth(m.captures[2])) end function alignment(io::IO, x::Complex) - m = match(r"^(.*[^ef][\+\-])(.*)$", sprint(show, x, context=io, sizehint=0)) - m === nothing ? (length(sprint(show, x, context=io, sizehint=0)), 0) : - (length(m.captures[1]), length(m.captures[2])) + s = sprint(show, x, context=nocolor(io), sizehint=0) + m = match(r"^(.*[^ef][\+\-])(.*)$", s) + m === nothing ? (textwidth(s), 0) : + (textwidth(m.captures[1]), textwidth(m.captures[2])) end function alignment(io::IO, x::Rational) - m = match(r"^(.*?/)(/.*)$", sprint(show, x, context=io, sizehint=0)) - m === nothing ? (length(sprint(show, x, context=io, sizehint=0)), 0) : - (length(m.captures[1]), length(m.captures[2])) + s = sprint(show, x, context=nocolor(io), sizehint=0) + m = match(r"^(.*?/)(/.*)$", s) + m === nothing ? (textwidth(s), 0) : + (textwidth(m.captures[1]), textwidth(m.captures[2])) end function alignment(io::IO, x::Pair) - s = sprint(show, x, context=io, sizehint=0) + fullwidth = alignment_from_show(io, x) if !isdelimited(io, x) # i.e. use "=>" for display ctx = IOContext(io, :typeinfo => gettypeinfos(io, x)[1]) - left = length(sprint(show, x.first, context=ctx, sizehint=0)) + left = alignment_from_show(ctx, x.first) left += 2 * !isdelimited(ctx, x.first) # for parens around p.first left += !(get(io, :compact, false)::Bool) # spaces are added around "=>" - (left+1, length(s)-left-1) # +1 for the "=" part of "=>" + (left+1, fullwidth-left-1) # +1 for the "=" part of "=>" else - (0, length(s)) # as for x::Any + (0, fullwidth) # as for x::Any end end diff --git a/test/show.jl b/test/show.jl index ba9f227e53e52..70dd9db898e75 100644 --- a/test/show.jl +++ b/test/show.jl @@ -2355,3 +2355,16 @@ end @test sprint(show, setenv(setcpuaffinity(`true`, [1, 2]), "A" => "B")) == """setenv(setcpuaffinity(`true`, [1, 2]),["A=B"])""" end + +# Test that alignment takes into account unicode and computes alignment without +# color/formatting. + +struct ColoredLetter; end +Base.show(io::IO, ces::ColoredLetter) = Base.printstyled(io, 'A'; color=:red) + +struct ⛵; end +Base.show(io::IO, ces::⛵) = Base.print(io, '⛵') + +@test Base.alignment(stdout, ⛵()) == (0, 2) +@test Base.alignment(IOContext(IOBuffer(), :color=>true), ColoredLetter()) == (0, 1) +@test Base.alignment(IOContext(IOBuffer(), :color=>false), ColoredLetter()) == (0, 1) From 431b19bc8941cd3bd722055d097233f4a70d1cab Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 1 Oct 2022 01:19:08 -0400 Subject: [PATCH 12/41] CI (Create Buildbot Statuses): remove `tester_win64` and `tester_win32` from the list (#46997) (cherry picked from commit 6424f790f41eba23c8bed79bc0a981775d876f9e) --- .github/workflows/statuses.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/statuses.yml b/.github/workflows/statuses.yml index 36a694a7c6d20..7116a766ad8df 100644 --- a/.github/workflows/statuses.yml +++ b/.github/workflows/statuses.yml @@ -48,8 +48,6 @@ jobs: - run: | declare -a CONTEXT_LIST=( "buildbot/tester_freebsd64" - "buildbot/tester_win32" - "buildbot/tester_win64" ) for CONTEXT in "${CONTEXT_LIST[@]}" do From 618bbc68676389aa88b7b5c1fd28281583e4b772 Mon Sep 17 00:00:00 2001 From: Christopher Rowley Date: Sun, 2 Oct 2022 11:18:03 +0100 Subject: [PATCH 13/41] make CanonicalIndexError an Exception type (#47008) (cherry picked from commit 7085703822767ea7621816c89b5746ab35dcbd20) --- base/abstractarray.jl | 2 +- test/abstractarray.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 864c0cc92dc5f..be31b406ae307 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1249,7 +1249,7 @@ function unsafe_getindex(A::AbstractArray, I...) r end -struct CanonicalIndexError +struct CanonicalIndexError <: Exception func::String type::Any CanonicalIndexError(func::String, @nospecialize(type)) = new(func, type) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index f504af8a08247..9d301f0eaa64b 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -525,6 +525,10 @@ function test_primitives(::Type{T}, shape, ::Type{TestAbstractArray}) where T @test_throws MethodError convert(Union{}, X) end +@testset "CanonicalIndexError is a Exception" begin + @test Base.CanonicalIndexError <: Exception +end + mutable struct TestThrowNoGetindex{T} <: AbstractVector{T} end @testset "ErrorException if getindex is not defined" begin Base.length(::TestThrowNoGetindex) = 2 From 5930711c96304b7302d35817828ef069b4f143a6 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Mon, 3 Oct 2022 06:02:17 +0600 Subject: [PATCH 14/41] =?UTF-8?q?Replace=20isfinite=20check=20in=20ranges?= =?UTF-8?q?=20with=20lo=20=E2=89=A4=20x=20=E2=89=A4=20hi=20(#45646)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Viral B. Shah (cherry picked from commit 58118253c63da7d67b9ddf326d3d3d47738e36d2) --- base/range.jl | 15 +++++++-------- test/ranges.jl | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/base/range.jl b/base/range.jl index d12a10518cd7f..404c8eaf5295a 100644 --- a/base/range.jl +++ b/base/range.jl @@ -1392,14 +1392,13 @@ function sum(r::AbstractRange{<:Real}) end function _in_range(x, r::AbstractRange) - if !isfinite(x) - return false - elseif iszero(step(r)) - return !isempty(r) && first(r) == x - else - n = round(Integer, (x - first(r)) / step(r)) + 1 - return n >= 1 && n <= length(r) && r[n] == x - end + isempty(r) && return false + f, l = first(r), last(r) + # check for NaN, Inf, and large x that may overflow in the next calculation + f <= x <= l || l <= x <= f || return false + iszero(step(r)) && return true + n = round(Integer, (x - f) / step(r)) + 1 + n >= 1 && n <= length(r) && r[n] == x end in(x::Real, r::AbstractRange{<:Real}) = _in_range(x, r) # This method needs to be defined separately since -(::T, ::T) can be implemented diff --git a/test/ranges.jl b/test/ranges.jl index c448f4b99e201..b68ab3312f304 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -518,8 +518,10 @@ end @test !(3.5 in 1:5) @test (3 in 1:5) @test (3 in 5:-1:1) - #@test (3 in 3+0*(1:5)) - #@test !(4 in 3+0*(1:5)) + @test (3 in 3 .+ 0*(1:5)) + @test !(4 in 3 .+ 0*(1:5)) + @test 0. in (0. .* (1:10)) + @test !(0.1 in (0. .* (1:10))) let r = 0.0:0.01:1.0 @test (r[30] in r) @@ -536,8 +538,17 @@ end x = (NaN16, Inf32, -Inf64, 1//0, -1//0) @test !(x in r) end + + @test 1e40 ∉ 0:1.0 # Issue #45747 + @test 1e20 ∉ 0:1e-20:1e-20 + @test 1e20 ∉ 0:1e-20 + @test 1.0 ∉ 0:1e-20:1e-20 + @test 0.5 ∉ 0:1e-20:1e-20 + @test 1 ∉ 0:1e-20:1e-20 + + @test_broken 17.0 ∈ 0:1e40 # Don't support really long ranges end - @testset "in() works across types, including non-numeric types (#21728)" begin + @testset "in() works across types, including non-numeric types (#21728 and #45646)" begin @test 1//1 in 1:3 @test 1//1 in 1.0:3.0 @test !(5//1 in 1:3) @@ -558,6 +569,22 @@ end @test !(Complex(1, 0) in Date(2017, 01, 01):Dates.Day(1):Date(2017, 01, 05)) @test !(π in Date(2017, 01, 01):Dates.Day(1):Date(2017, 01, 05)) @test !("a" in Date(2017, 01, 01):Dates.Day(1):Date(2017, 01, 05)) + + # We use Ducks because of their propensity to stand in a row and because we know + # that no additional methods (e.g. isfinite) are defined specifically for Ducks. + struct Duck + location::Int + end + Base.:+(x::Duck, y::Int) = Duck(x.location + y) + Base.:-(x::Duck, y::Int) = Duck(x.location - y) + Base.:-(x::Duck, y::Duck) = x.location - y.location + Base.isless(x::Duck, y::Duck) = isless(x.location, y.location) + + @test Duck(3) ∈ Duck(1):2:Duck(5) + @test Duck(3) ∈ Duck(5):-2:Duck(2) + @test Duck(4) ∉ Duck(5):-2:Duck(1) + @test Duck(4) ∈ Duck(1):Duck(5) + @test Duck(0) ∉ Duck(1):Duck(5) end end @testset "indexing range with empty range (#4309)" begin From e7b0555763c22cd2b32983360d5e745d402ffc44 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Thu, 6 Oct 2022 12:10:25 -0700 Subject: [PATCH 15/41] gc: add missing root for binding->ty field (#46806) Co-authored-by: Jameson Nash (cherry picked from commit 45b96c4f1ae03dc7f6bf227e50cd2b44dad5b5c4) --- src/gc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/gc.c b/src/gc.c index b60cc4ff7e8d6..893b76139b7e6 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2457,6 +2457,16 @@ module_binding: { void *vb = jl_astaggedvalue(b); verify_parent1("module", binding->parent, &vb, "binding_buff"); (void)vb; + jl_value_t *ty = jl_atomic_load_relaxed(&b->ty); + if (ty && ty != (jl_value_t*)jl_any_type) { + verify_parent2("module", binding->parent, + &b->ty, "binding(%s)", jl_symbol_name(b->name)); + if (gc_try_setmark(ty, &binding->nptr, &tag, &bits)) { + new_obj = ty; + gc_repush_markdata(&sp, gc_mark_binding_t); + goto mark; + } + } jl_value_t *value = jl_atomic_load_relaxed(&b->value); jl_value_t *globalref = jl_atomic_load_relaxed(&b->globalref); if (value) { From 4da1a66bdd84402ae9d583c6d5cab63edc39b0c2 Mon Sep 17 00:00:00 2001 From: Daniel Karrasch Date: Mon, 10 Oct 2022 04:10:21 +0200 Subject: [PATCH 16/41] Check sizes in 3-arg diagonal (dot-)product (#47114) (cherry picked from commit 25e3809ea4c309f7d0d9d5db85c5107aa63877b5) --- stdlib/LinearAlgebra/src/diagonal.jl | 3 +++ stdlib/LinearAlgebra/test/diagonal.jl | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/stdlib/LinearAlgebra/src/diagonal.jl b/stdlib/LinearAlgebra/src/diagonal.jl index 4af42d8f53eb4..a2756bb3a1201 100644 --- a/stdlib/LinearAlgebra/src/diagonal.jl +++ b/stdlib/LinearAlgebra/src/diagonal.jl @@ -770,6 +770,9 @@ end dot(A::AbstractMatrix, B::Diagonal) = conj(dot(B, A)) function _mapreduce_prod(f, x, D::Diagonal, y) + if !(length(x) == length(D.diag) == length(y)) + throw(DimensionMismatch("x has length $(length(x)), D has size $(size(D)), and y has $(length(y))")) + end if isempty(x) && isempty(D) && isempty(y) return zero(promote_op(f, eltype(x), eltype(D), eltype(y))) else diff --git a/stdlib/LinearAlgebra/test/diagonal.jl b/stdlib/LinearAlgebra/test/diagonal.jl index 2801332e840e6..007420f1eb999 100644 --- a/stdlib/LinearAlgebra/test/diagonal.jl +++ b/stdlib/LinearAlgebra/test/diagonal.jl @@ -922,10 +922,14 @@ end @test s1 == prod(sign, d) end -@testset "Empty (#35424)" begin +@testset "Empty (#35424) & size checks (#47060)" begin @test zeros(0)'*Diagonal(zeros(0))*zeros(0) === 0.0 @test transpose(zeros(0))*Diagonal(zeros(Complex{Int}, 0))*zeros(0) === 0.0 + 0.0im @test dot(zeros(Int32, 0), Diagonal(zeros(Int, 0)), zeros(Int16, 0)) === 0 + @test_throws DimensionMismatch zeros(2)' * Diagonal(zeros(2)) * zeros(3) + @test_throws DimensionMismatch zeros(3)' * Diagonal(zeros(2)) * zeros(2) + @test_throws DimensionMismatch dot(zeros(2), Diagonal(zeros(2)), zeros(3)) + @test_throws DimensionMismatch dot(zeros(3), Diagonal(zeros(2)), zeros(2)) end @testset "Diagonal(undef)" begin From 1e78c646c48a96a1337f44771438c5147e38c6aa Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 10 Oct 2022 12:58:11 +0200 Subject: [PATCH 17/41] make REPL completions robust against `expanduser` throwing (#47058) (cherry picked from commit 7beeaf70e412cd09e4e7d639313388aa25062936) --- stdlib/REPL/src/REPLCompletions.jl | 7 ++++++- stdlib/REPL/test/replcompletions.jl | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 170ce53061a3a..ca00da2525c5a 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -312,7 +312,12 @@ function complete_path(path::AbstractString, pos::Int; use_envpath=false, shell_ end function complete_expanduser(path::AbstractString, r) - expanded = expanduser(path) + expanded = + try expanduser(path) + catch e + e isa ArgumentError || rethrow() + path + end return Completion[PathCompletion(expanded)], r, path != expanded end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 3d88b326d94a4..fa9f40f626f67 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -953,6 +953,9 @@ let s, c, r s = "\"~" @test "tmpfoobar/" in c c,r = test_complete(s) + s = "\"~user" + c, r = test_complete(s) + @test isempty(c) rm(dir) end end From aa2d7c47039052c2ca1ec62d8a038cc297bc5d80 Mon Sep 17 00:00:00 2001 From: Hendrik Ranocha Date: Mon, 10 Oct 2022 13:30:58 +0200 Subject: [PATCH 18/41] fix invalidations when loading ForwardDiff.jl (#47091) (cherry picked from commit 0dada1752f1a1f022b6566c69e872dd71b37fc98) --- base/namedtuple.jl | 2 +- stdlib/InteractiveUtils/src/macros.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/namedtuple.jl b/base/namedtuple.jl index 2157975ba30d1..7a08e4f540b51 100644 --- a/base/namedtuple.jl +++ b/base/namedtuple.jl @@ -111,7 +111,7 @@ function NamedTuple{names}(nt::NamedTuple) where {names} types = Tuple{(fieldtype(nt, idx[n]) for n in 1:length(idx))...} Expr(:new, :(NamedTuple{names, $types}), Any[ :(getfield(nt, $(idx[n]))) for n in 1:length(idx) ]...) else - length_names = length(names)::Integer + length_names = length(names::Tuple) types = Tuple{(fieldtype(typeof(nt), names[n]) for n in 1:length_names)...} NamedTuple{names, types}(map(Fix1(getfield, nt), names)) end diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index d9dcff57257c9..5d1e6dff77478 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -24,7 +24,7 @@ function recursive_dotcalls!(ex, args, i=1) end end (start, branches) = ex.head === :. ? (1, ex.args[2].args) : (2, ex.args) - length_branches = length(branches)::Integer + length_branches = length(branches)::Int for j in start:length_branches branch, i = recursive_dotcalls!(branches[j], args, i) branches[j] = branch @@ -40,7 +40,7 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) end i = findlast(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args[1].args) args = copy(ex0.args[1].args) - insert!(args, (isnothing(i) ? 2 : i+1), ex0.args[2]) + insert!(args, (isnothing(i) ? 2 : 1+i::Int), ex0.args[2]) ex0 = Expr(:call, args...) end if ex0.head === :. || (ex0.head === :call && ex0.args[1] !== :.. && string(ex0.args[1])[1] == '.') From 7b1c32dbf1b9a535ad81d3b9b150e5495e019038 Mon Sep 17 00:00:00 2001 From: William Moses Date: Mon, 10 Oct 2022 14:09:38 -0400 Subject: [PATCH 19/41] optimizer: fix alloc opt on unknown offset with references (#47076) Fixes issued mentioned in https://github.com/JuliaLang/julia/issues/47075#issuecomment-1269283426 (cherry picked from commit a68235c9b48e8b5368e85397ebb76dc348b6b55d) --- src/llvm-alloc-helpers.cpp | 14 +++++++++-- src/llvm-alloc-helpers.h | 7 ++++++ src/llvm-alloc-opt.cpp | 8 ++++--- test/llvmpasses/alloc-opt-unsized.ll | 35 ++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 test/llvmpasses/alloc-opt-unsized.ll diff --git a/src/llvm-alloc-helpers.cpp b/src/llvm-alloc-helpers.cpp index 55a93ea5179b5..886427ed01145 100644 --- a/src/llvm-alloc-helpers.cpp +++ b/src/llvm-alloc-helpers.cpp @@ -161,7 +161,12 @@ void jl_alloc::runEscapeAnalysis(llvm::Instruction *I, EscapeAnalysisRequiredArg auto check_inst = [&] (Instruction *inst, Use *use) { if (isa(inst)) { required.use_info.hasload = true; - if (cur.offset == UINT32_MAX || !required.use_info.addMemOp(inst, 0, cur.offset, + if (cur.offset == UINT32_MAX) { + auto elty = inst->getType(); + required.use_info.has_unknown_objref |= hasObjref(elty); + required.use_info.has_unknown_objrefaggr |= hasObjref(elty) && !isa(elty); + required.use_info.hasunknownmem = true; + } else if (!required.use_info.addMemOp(inst, 0, cur.offset, inst->getType(), false, required.DL)) required.use_info.hasunknownmem = true; @@ -229,7 +234,12 @@ void jl_alloc::runEscapeAnalysis(llvm::Instruction *I, EscapeAnalysisRequiredArg return false; } auto storev = store->getValueOperand(); - if (cur.offset == UINT32_MAX || !required.use_info.addMemOp(inst, use->getOperandNo(), + if (cur.offset == UINT32_MAX) { + auto elty = storev->getType(); + required.use_info.has_unknown_objref |= hasObjref(elty); + required.use_info.has_unknown_objrefaggr |= hasObjref(elty) && !isa(elty); + required.use_info.hasunknownmem = true; + } else if (!required.use_info.addMemOp(inst, use->getOperandNo(), cur.offset, storev->getType(), true, required.DL)) required.use_info.hasunknownmem = true; diff --git a/src/llvm-alloc-helpers.h b/src/llvm-alloc-helpers.h index 3f06baddfcff6..b0c675ae59cc8 100644 --- a/src/llvm-alloc-helpers.h +++ b/src/llvm-alloc-helpers.h @@ -87,6 +87,11 @@ namespace jl_alloc { // The object is used in an error function bool haserror:1; + // The alloc has a Julia object reference not in an explicit field. + bool has_unknown_objref:1; + // The alloc has an aggregate Julia object reference not in an explicit field. + bool has_unknown_objrefaggr:1; + void reset() { escaped = false; @@ -99,6 +104,8 @@ namespace jl_alloc { hasunknownmem = false; returned = false; haserror = false; + has_unknown_objref = false; + has_unknown_objrefaggr = false; uses.clear(); preserves.clear(); memops.clear(); diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index 2347b51c899c9..d9ddc7a15b9fb 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -233,8 +233,8 @@ void Optimizer::optimizeAll() removeAlloc(orig); continue; } - bool has_ref = false; - bool has_refaggr = false; + bool has_ref = use_info.has_unknown_objref; + bool has_refaggr = use_info.has_unknown_objrefaggr; for (auto memop: use_info.memops) { auto &field = memop.second; if (field.hasobjref) { @@ -577,7 +577,9 @@ void Optimizer::moveToStack(CallInst *orig_inst, size_t sz, bool has_ref) // treat this as a non-mem2reg'd alloca // The ccall root and GC preserve handling below makes sure that // the alloca isn't optimized out. - buff = prolog_builder.CreateAlloca(pass.T_prjlvalue); + const DataLayout &DL = F.getParent()->getDataLayout(); + auto asize = ConstantInt::get(Type::getInt64Ty(prolog_builder.getContext()), sz / DL.getTypeAllocSize(pass.T_prjlvalue)); + buff = prolog_builder.CreateAlloca(pass.T_prjlvalue, asize); buff->setAlignment(Align(align)); ptr = cast(prolog_builder.CreateBitCast(buff, pass.T_pint8)); } diff --git a/test/llvmpasses/alloc-opt-unsized.ll b/test/llvmpasses/alloc-opt-unsized.ll new file mode 100644 index 0000000000000..f7ea31fde6b05 --- /dev/null +++ b/test/llvmpasses/alloc-opt-unsized.ll @@ -0,0 +1,35 @@ +; RUN: opt -enable-new-pm=0 -load libjulia-codegen%shlibext -AllocOpt -S %s | FileCheck %s + +source_filename = "text" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:10:11:12:13" +target triple = "x86_64-linux-gnu" + +declare {}*** @julia.get_pgcstack() + +declare {} addrspace(10)* @julia.gc_alloc_obj({}**, i64, {} addrspace(10)*) + +declare void @julia.write_barrier({} addrspace(10)*, ...) + +define void @diffejulia_objective__1864_inner_1wrap({} addrspace(10)* %arg, i64 %iv.i) { +entry: + %i5 = call {}*** @julia.get_pgcstack() + %i13 = bitcast {}*** %i5 to {}** + %i14 = getelementptr inbounds {}*, {}** %i13, i64 -12 + %i18 = call noalias nonnull dereferenceable(8000) dereferenceable_or_null(8000) {} addrspace(10)* @julia.gc_alloc_obj({}** %i14, i64 8000, {} addrspace(10)* addrspacecast ({}* inttoptr (i64 139756155247504 to {}*) to {} addrspace(10)*)) + %_malloccache.i = bitcast {} addrspace(10)* %i18 to {} addrspace(10)* addrspace(10)* + %i23 = getelementptr inbounds {} addrspace(10)*, {} addrspace(10)* addrspace(10)* %_malloccache.i, i64 %iv.i + store {} addrspace(10)* %arg, {} addrspace(10)* addrspace(10)* %i23, align 8 + %i24 = bitcast {} addrspace(10)* addrspace(10)* %_malloccache.i to {} addrspace(10)* + call void ({} addrspace(10)*, ...) @julia.write_barrier({} addrspace(10)* %i24, {} addrspace(10)* %arg) + %l = load {} addrspace(10)*, {} addrspace(10)* addrspace(10)* %i23 + ret void +} + +; CHECK: %[[i0:.+]] = alloca {} addrspace(10)*, i64 1000, align 16 +; CHECK: %[[i1:.+]] = bitcast {} addrspace(10)** %[[i0]] to i8* +; CHECK: %i18 = bitcast i8* %[[i1]] to {}* +; CHECK: %_malloccache.i = bitcast {}* %i18 to {} addrspace(10)** +; CHECK: %i23 = getelementptr inbounds {} addrspace(10)*, {} addrspace(10)** %_malloccache.i, i64 %iv.i +; CHECK: store {} addrspace(10)* %arg, {} addrspace(10)** %i23, align 8 +; CHECK: %i24 = bitcast {} addrspace(10)** %_malloccache.i to {}* +; CHECK: %l = load {} addrspace(10)*, {} addrspace(10)** %i23, align 8 From 0c0c86c2faf73cf2606536e24b5df5a5d01cd222 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 17 Oct 2022 11:17:06 -0400 Subject: [PATCH 20/41] fix spelling of :not_atomic symbol default value (#47187) (cherry picked from commit dc3a2e8d27a6527b3ca3583e478710dffce18319) --- base/Base.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index b2e79224edbd7..79c6d64c001ee 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -50,11 +50,11 @@ setproperty!(x::Tuple, f::Int, v, order::Symbol) = setfield!(x, f, v, order) # t getproperty(x, f::Symbol, order::Symbol) = (@inline; getfield(x, f, order)) setproperty!(x, f::Symbol, v, order::Symbol) = (@inline; setfield!(x, f, convert(fieldtype(typeof(x), f), v), order)) -swapproperty!(x, f::Symbol, v, order::Symbol=:notatomic) = +swapproperty!(x, f::Symbol, v, order::Symbol=:not_atomic) = (@inline; Core.swapfield!(x, f, convert(fieldtype(typeof(x), f), v), order)) -modifyproperty!(x, f::Symbol, op, v, order::Symbol=:notatomic) = +modifyproperty!(x, f::Symbol, op, v, order::Symbol=:not_atomic) = (@inline; Core.modifyfield!(x, f, op, v, order)) -replaceproperty!(x, f::Symbol, expected, desired, success_order::Symbol=:notatomic, fail_order::Symbol=success_order) = +replaceproperty!(x, f::Symbol, expected, desired, success_order::Symbol=:not_atomic, fail_order::Symbol=success_order) = (@inline; Core.replacefield!(x, f, expected, convert(fieldtype(typeof(x), f), desired), success_order, fail_order)) convert(::Type{Any}, Core.@nospecialize x) = x From 0da955a0107b215b2fbf8592f56eb03d88a767f0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Oct 2022 13:06:32 -0400 Subject: [PATCH 21/41] fix linkage of libuv (#47203) Only libjulia-internal is supposed to be linked against this, since it is a static library. Fix #47198 (cherry picked from commit e8aacc812bf022f6da91bd24bbcf9923be17763b) --- src/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index 475f2eb949d6e..8ef266c8e42d3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -154,12 +154,12 @@ OSLIBS += $(SRCDIR)/mach_dyld_atfork.tbd endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) -RT_LIBS := $(LIBUV) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) -CG_LIBS := $(NO_WHOLE_ARCHIVE) $(LIBUV) $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) +RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) +CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) RT_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a -ljulia-debug $(RT_LIBS) -CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(WHOLE_ARCHIVE) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug +CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug RT_RELEASE_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a -ljulia $(RT_LIBS) -CG_RELEASE_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(WHOLE_ARCHIVE) $(CG_LIBS) -ljulia -ljulia-internal +CG_RELEASE_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia -ljulia-internal OBJS := $(SRCS:%=$(BUILDDIR)/%.o) DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) From 7d8f474059b829f25cf56750e2ee8b2f639832ce Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Wed, 19 Oct 2022 20:44:43 -0600 Subject: [PATCH 22/41] Ensure port in listenany stays UInt16 (#47213) As reported [here](https://discourse.julialang.org/t/test-failures-for-sockets-base-runtests-sockets/88898). My guess on the original issue reported is that, for some reason, the host where the tests are run is unable to listen on any ports, so we end up cycling through the entire UInt16 range (the test starts at port 11011), but when we fail on port 65535, we do `addr.port + 1` and instead of wrapping around as I believe this function intends to happen (as noted by the `addr.port == default_port` check before we error), it gets promoted to `Int(65536)` which then throws an (unexpected) error in the `InetAddr` constructor. I'm not quite sure how to test this exactly though, because we'd need to simulate not being able to listen on any ports? If anyone has any ideas, I'm all ears. (cherry picked from commit a311f4d8327a5051b11a6bcd1c44ed931d4ab261) --- stdlib/Sockets/src/Sockets.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Sockets/src/Sockets.jl b/stdlib/Sockets/src/Sockets.jl index 84fe351de99e1..9dc630c2db9fd 100644 --- a/stdlib/Sockets/src/Sockets.jl +++ b/stdlib/Sockets/src/Sockets.jl @@ -727,7 +727,7 @@ function listenany(host::IPAddr, default_port) return (addr.port, sock) end close(sock) - addr = InetAddr(addr.host, addr.port + 1) + addr = InetAddr(addr.host, addr.port + UInt16(1)) if addr.port == default_port error("no ports available") end From b8ab6cf2304bc83331787c4eaef3fad190617a91 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 24 Oct 2022 15:38:42 -0400 Subject: [PATCH 23/41] [InteractiveUtils] win: handle empty clipboard errors (#47311) Fix #46981 (cherry picked from commit 064542706ece6159dd18a6907a278f548c63faaa) --- stdlib/InteractiveUtils/src/clipboard.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stdlib/InteractiveUtils/src/clipboard.jl b/stdlib/InteractiveUtils/src/clipboard.jl index 7bc718b91b2bd..27b82343973e4 100644 --- a/stdlib/InteractiveUtils/src/clipboard.jl +++ b/stdlib/InteractiveUtils/src/clipboard.jl @@ -103,7 +103,7 @@ elseif Sys.iswindows() ccall(:memcpy, Ptr{UInt16}, (Ptr{UInt16}, Ptr{UInt16}, Csize_t), plock, x_u16, sizeof(x_u16)) unlock = ccall((:GlobalUnlock, "kernel32"), stdcall, Cint, (Ptr{UInt16},), pdata) (unlock == 0 && Libc.GetLastError() == 0) || return cleanup(:GlobalUnlock) # this should never fail - pset = ccall((:SetClipboardData, "user32"), stdcall, Ptr{UInt16}, (Cuint, Ptr{UInt16}), 13, pdata) + pset = ccall((:SetClipboardData, "user32"), stdcall, Ptr{UInt16}, (Cuint, Ptr{UInt16}), 13, pdata) # CF_UNICODETEXT pdata != pset && return cleanup(:SetClipboardData) cleanup(:success) end @@ -114,14 +114,14 @@ elseif Sys.iswindows() if cause !== :OpenClipboard ccall((:CloseClipboard, "user32"), stdcall, Cint, ()) == 0 && Base.windowserror(:CloseClipboard) # this should never fail end - if cause !== :success && (cause !== :GetClipboardData || errno != 0) + if cause !== :success && !(cause === :GetClipboardData && (errno == 0x8004006A || errno == 0x800401D3)) # ignore DV_E_CLIPFORMAT and CLIPBRD_E_BAD_DATA from GetClipboardData Base.windowserror(cause, errno) end "" end ccall((:OpenClipboard, "user32"), stdcall, Cint, (Ptr{Cvoid},), C_NULL) == 0 && return Base.windowserror(:OpenClipboard) ccall(:SetLastError, stdcall, Cvoid, (UInt32,), 0) # allow distinguishing if the clipboard simply didn't have text - pdata = ccall((:GetClipboardData, "user32"), stdcall, Ptr{UInt16}, (Cuint,), 13) + pdata = ccall((:GetClipboardData, "user32"), stdcall, Ptr{UInt16}, (Cuint,), 13) # CF_UNICODETEXT pdata == C_NULL && return cleanup(:GetClipboardData) plock = ccall((:GlobalLock, "kernel32"), stdcall, Ptr{UInt16}, (Ptr{UInt16},), pdata) plock == C_NULL && return cleanup(:GlobalLock) From 8c8b6f0db2c8d479cc608a5379011271ec148953 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Tue, 25 Oct 2022 22:15:02 +0200 Subject: [PATCH 24/41] Better Libdl.dlopen error when using non-standard extension (#46998) When trying to dlopen a file with non-standard extension (e.g. `foo.so` instead of `foo.dylib` when running on macOS), if this failed (e.g. because of a reference to an undefined symbol), then instead of printing the error message returned by `dlerror` with a helpful notice what went wrong, a message indicating something to the effect that "foo.so.dylib was not found" was shown, which was not helpful at all. To get the actual helpful error message, add a check so that when dlopen fails for a file that actually exists, we don't retry loading from a file with the standard extension attached, which might not even exist; instead we just give up. This matches what is already being done for relative paths. This patch simply copies the relevant check to also be applied to the case dealing with absolute paths. (cherry picked from commit a490197b4dc6d4f94475f2f644942eb2db560a50) --- src/dlload.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dlload.c b/src/dlload.c index 33afe62acad90..8932420e2af14 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -259,6 +259,10 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, #ifdef _OS_WINDOWS_ err = GetLastError(); break; // LoadLibrary already tested the rest +#else + // bail out and show the error if file actually exists + if (jl_stat(path, (char*)&stbuf) == 0) + break; #endif } From e4b0dfb5d57b8f1267d53bf8651ba35891a4ccc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Fri, 1 Apr 2022 18:59:06 +0100 Subject: [PATCH 25/41] Mark as `inline` functions implemented in `src/serialize.h` (#44739) * Mark as `inline` functions implemented in `src/serialize.h` Also, remove duplicate definitions from `src/staticdata.c` and include `serialize.h` in there. (cherry picked from commit 081ae64f7098a7d1a8bfb9162b62115c09de2742) --- src/serialize.h | 20 ++++++++++---------- src/staticdata.c | 20 +------------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 817591b989f93..564600974d1d6 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -67,25 +67,25 @@ extern "C" { #define LAST_TAG 57 #define write_uint8(s, n) ios_putc((n), (s)) -#define read_uint8(s) ((uint8_t)ios_getc(s)) -#define write_int8(s, n) write_uint8(s, n) -#define read_int8(s) read_uint8(s) +#define read_uint8(s) ((uint8_t)ios_getc((s))) +#define write_int8(s, n) write_uint8((s), (n)) +#define read_int8(s) read_uint8((s)) /* read and write in host byte order */ -static void write_int32(ios_t *s, int32_t i) JL_NOTSAFEPOINT +static inline void write_int32(ios_t *s, int32_t i) JL_NOTSAFEPOINT { ios_write(s, (char*)&i, 4); } -static int32_t read_int32(ios_t *s) JL_NOTSAFEPOINT +static inline int32_t read_int32(ios_t *s) JL_NOTSAFEPOINT { int32_t x = 0; ios_read(s, (char*)&x, 4); return x; } -static uint64_t read_uint64(ios_t *s) JL_NOTSAFEPOINT +static inline uint64_t read_uint64(ios_t *s) JL_NOTSAFEPOINT { uint64_t x = 0; ios_read(s, (char*)&x, 8); @@ -97,24 +97,24 @@ static void write_int64(ios_t *s, int64_t i) JL_NOTSAFEPOINT ios_write(s, (char*)&i, 8); } -static void write_uint16(ios_t *s, uint16_t i) JL_NOTSAFEPOINT +static inline void write_uint16(ios_t *s, uint16_t i) JL_NOTSAFEPOINT { ios_write(s, (char*)&i, 2); } -static uint16_t read_uint16(ios_t *s) JL_NOTSAFEPOINT +static inline uint16_t read_uint16(ios_t *s) JL_NOTSAFEPOINT { int16_t x = 0; ios_read(s, (char*)&x, 2); return x; } -static void write_uint32(ios_t *s, uint32_t i) JL_NOTSAFEPOINT +static inline void write_uint32(ios_t *s, uint32_t i) JL_NOTSAFEPOINT { ios_write(s, (char*)&i, 4); } -static uint32_t read_uint32(ios_t *s) JL_NOTSAFEPOINT +static inline uint32_t read_uint32(ios_t *s) JL_NOTSAFEPOINT { uint32_t x = 0; ios_read(s, (char*)&x, 4); diff --git a/src/staticdata.c b/src/staticdata.c index 1b98601026d49..827afac81b6da 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -65,6 +65,7 @@ done by `get_item_for_reloc`. #include "julia_internal.h" #include "builtin_proto.h" #include "processor.h" +#include "serialize.h" #ifndef _OS_WINDOWS_ #include @@ -361,25 +362,6 @@ typedef enum { // if a larger size is required, will need to add support for writing larger relocations in many cases below #define RELOC_TAG_OFFSET 29 - -/* read and write in host byte order */ - -#define write_uint8(s, n) ios_putc((n), (s)) -#define read_uint8(s) ((uint8_t)ios_getc((s))) - -static void write_uint32(ios_t *s, uint32_t i) JL_NOTSAFEPOINT -{ - ios_write(s, (char*)&i, 4); -} - -static uint32_t read_uint32(ios_t *s) JL_NOTSAFEPOINT -{ - uint32_t x = 0; - ios_read(s, (char*)&x, 4); - return x; -} - - // --- Static Compile --- static void *jl_sysimg_handle = NULL; From baa7001506a1bda796a717d23b106a959948aae7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 21 Sep 2022 13:27:34 +0200 Subject: [PATCH 26/41] ccall: handle Union appearing as a field-type without crashing (#46787) We disallow union as the direct type, so perhaps we should disallow it as a field-type also, but since we do allow references in those cases typically, we will allow this also. Also DRY the emit_global code, since it had bit-rotted relative to the usual code path through emit_globalref (and apparently could still be run though for handling the first argument to cfunction). Fix #46786 (cherry picked from commit ed01ee036b0c63e698f40b2373c0e7fe704d6e35) --- src/abi_aarch64.cpp | 18 ++++++++++++------ src/abi_arm.cpp | 2 ++ src/abi_ppc64le.cpp | 7 +++++-- src/abi_x86_64.cpp | 4 ++++ src/codegen.cpp | 21 +-------------------- test/ccall.jl | 26 ++++++++++++++++++++++++++ 6 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/abi_aarch64.cpp b/src/abi_aarch64.cpp index 1a3f160329c6c..514c3c5a81a6d 100644 --- a/src/abi_aarch64.cpp +++ b/src/abi_aarch64.cpp @@ -43,9 +43,11 @@ Type *get_llvm_vectype(jl_datatype_t *dt, LLVMContext &ctx) const // the homogeneity check. jl_datatype_t *ft0 = (jl_datatype_t*)jl_field_type(dt, 0); // `ft0` should be a `VecElement` type and the true element type - // should be a primitive type - if (ft0->name != jl_vecelement_typename || - ((jl_datatype_t*)jl_field_type(ft0, 0))->layout->nfields) + // should be a primitive type (nfields == 0) + if (!jl_is_datatype(ft0) || ft0->name != jl_vecelement_typename) + return nullptr; + jl_datatype_t *ft00 = (jl_datatype_t*)jl_field_type(ft0, 0); + if (!jl_is_datatype(ft00) || ft00->layout->nfields) return nullptr; for (size_t i = 1; i < nfields; i++) { if (jl_field_type(dt, i) != (jl_value_t*)ft0) { @@ -120,15 +122,17 @@ bool isHFAorHVA(jl_datatype_t *dt, size_t dsz, size_t &nele, ElementType &ele, L // For composite types, find the first non zero sized member size_t i; size_t fieldsz; - for (i = 0;i < nfields;i++) { + for (i = 0; i < nfields; i++) { if ((fieldsz = jl_field_size(dt, i))) { break; } } assert(i < nfields); - // If there's only one non zero sized member, try again on this member + // If there's only one non-zero sized member, try again on this member if (fieldsz == dsz) { dt = (jl_datatype_t*)jl_field_type(dt, i); + if (!jl_is_datatype(dt)) + return false; continue; } if (Type *vectype = get_llvm_vectype(dt, ctx)) { @@ -140,11 +144,13 @@ bool isHFAorHVA(jl_datatype_t *dt, size_t dsz, size_t &nele, ElementType &ele, L return true; } // Otherwise, process each members - for (;i < nfields;i++) { + for (; i < nfields; i++) { size_t fieldsz = jl_field_size(dt, i); if (fieldsz == 0) continue; jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type(dt, i); + if (!jl_is_datatype(dt)) + return false; // Check element count. // This needs to be done after the zero size member check if (nele > 3 || !isHFAorHVA(fieldtype, fieldsz, nele, ele, ctx)) { diff --git a/src/abi_arm.cpp b/src/abi_arm.cpp index 4987d07657ae6..441aa95b1fdf6 100644 --- a/src/abi_arm.cpp +++ b/src/abi_arm.cpp @@ -91,6 +91,8 @@ size_t isLegalHA(jl_datatype_t *dt, Type *&base, LLVMContext &ctx) const size_t parent_members = jl_datatype_nfields(dt); for (size_t i = 0; i < parent_members; ++i) { jl_datatype_t *fdt = (jl_datatype_t*)jl_field_type(dt,i); + if (!jl_is_datatype(fdt)) + return 0; Type *T = isLegalHAType(fdt, ctx); if (T) diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index 016eebd455525..2e18acdbd4f4b 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -44,6 +44,9 @@ struct ABI_PPC64leLayout : AbiLayout { // count the homogeneous floating aggregate size (saturating at max count of 8) unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) const { + if (jl_datatype_size(ty) > 128 || ty->layout->npointers || ty->layout->haspadding) + return 9; + size_t i, l = ty->layout->nfields; // handle homogeneous float aggregates if (l == 0) { @@ -52,7 +55,7 @@ unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) const *hva = false; if (*ty0 == NULL) *ty0 = ty; - else if (*hva || ty->size != (*ty0)->size) + else if (*hva || jl_datatype_size(ty) != jl_datatype_size(*ty0)) return 9; return 1; } @@ -69,7 +72,7 @@ unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) const *hva = true; if (*ty0 == NULL) *ty0 = ty; - else if (!*hva || ty->size != (*ty0)->size) + else if (!*hva || jl_datatype_size(ty) != jl_datatype_size(*ty0)) return 9; for (i = 1; i < l; i++) { jl_datatype_t *fld = (jl_datatype_t*)jl_field_type(ty, i); diff --git a/src/abi_x86_64.cpp b/src/abi_x86_64.cpp index 43e539b8386ce..c3d12417e6de8 100644 --- a/src/abi_x86_64.cpp +++ b/src/abi_x86_64.cpp @@ -153,6 +153,10 @@ void classifyType(Classification& accum, jl_datatype_t *dt, uint64_t offset) con jl_value_t *ty = jl_field_type(dt, i); if (jl_field_isptr(dt, i)) ty = (jl_value_t*)jl_voidpointer_type; + else if (!jl_is_datatype(ty)) { // inline union + accum.addField(offset, Memory); + continue; + } classifyType(accum, (jl_datatype_t*)ty, offset + jl_field_offset(dt, i)); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index c7aba9a4942cd..104f77f8d65ab 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3955,25 +3955,6 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i) return mark_julia_type(ctx, sp, true, jl_any_type); } -static jl_cgval_t emit_global(jl_codectx_t &ctx, jl_sym_t *sym) -{ - jl_binding_t *jbp = NULL; - Value *bp = global_binding_pointer(ctx, ctx.module, sym, &jbp, false); - assert(bp != NULL); - if (jbp && jbp->value != NULL) { - if (jbp->constp) - return mark_julia_const(ctx, jbp->value); - // double-check that a global variable is actually defined. this - // can be a problem in parallel when a definition is missing on - // one machine. - LoadInst *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*))); - v->setOrdering(AtomicOrdering::Unordered); - tbaa_decorate(ctx.tbaa().tbaa_binding, v); - return mark_julia_type(ctx, v, true, jl_any_type); - } - return emit_checked_var(ctx, bp, sym, false, ctx.tbaa().tbaa_binding); -} - static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) { Value *isnull = NULL; @@ -4626,7 +4607,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) { if (jl_is_symbol(expr)) { jl_sym_t *sym = (jl_sym_t*)expr; - return emit_global(ctx, sym); + return emit_globalref(ctx, ctx.module, sym); } if (jl_is_slot(expr) || jl_is_argument(expr)) { return emit_local(ctx, expr); diff --git a/test/ccall.jl b/test/ccall.jl index 3a1b6ff3db733..f0d240aad342d 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -1590,6 +1590,32 @@ function caller22734(ptr) end @test caller22734(ptr22734) === 32.0 +# issue #46786 -- non-isbitstypes passed "by-value" +struct NonBits46786 + x::Union{Int16,NTuple{3,UInt8}} +end +let ptr = @cfunction(identity, NonBits46786, (NonBits46786,)) + obj1 = NonBits46786((0x01,0x02,0x03)) + obj2 = ccall(ptr, NonBits46786, (NonBits46786,), obj1) + @test obj1 === obj2 +end +let ptr = @cfunction(identity, Base.RefValue{NonBits46786}, (Base.RefValue{NonBits46786},)) + obj1 = Base.RefValue(NonBits46786((0x01,0x02,0x03))) + obj2 = ccall(ptr, Base.RefValue{NonBits46786}, (Base.RefValue{NonBits46786},), obj1) + @test obj1 !== obj2 + @test obj1.x === obj2.x +end + +mutable struct MutNonBits46786 + x::Union{Int16,NTuple{3,UInt8}} +end +let ptr = @cfunction(identity, MutNonBits46786, (MutNonBits46786,)) + obj1 = MutNonBits46786((0x01,0x02,0x03)) + obj2 = ccall(ptr, MutNonBits46786, (MutNonBits46786,), obj1) + @test obj1 !== obj2 + @test obj1.x === obj2.x +end + # 26297#issuecomment-371165725 # test that the first argument to cglobal is recognized as a tuple literal even through # macro expansion From d21a9f2f3c6496f417679839c45140dab9f9d31d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 22 Sep 2022 18:33:49 +0200 Subject: [PATCH 27/41] staticdata: avoid needing unaligned loads, support 64-bit images (#46683) No functional changes intended here. (cherry picked from commit 45623a8814f6808273c67e698352b8d077673ccf) --- src/dump.c | 11 +-- src/ircode.c | 4 +- src/serialize.h | 15 ++++- src/staticdata.c | 171 +++++++++++++++++++++++++++-------------------- 4 files changed, 118 insertions(+), 83 deletions(-) diff --git a/src/dump.c b/src/dump.c index dfd80f2a5fc63..920a6981db095 100644 --- a/src/dump.c +++ b/src/dump.c @@ -120,11 +120,6 @@ static jl_typename_t *jl_idtable_typename = NULL; static jl_value_t *jl_bigint_type = NULL; static int gmp_limb_size = 0; -static void write_uint64(ios_t *s, uint64_t i) JL_NOTSAFEPOINT -{ - ios_write(s, (char*)&i, 8); -} - static void write_float64(ios_t *s, double x) JL_NOTSAFEPOINT { write_uint64(s, *((uint64_t*)&x)); @@ -926,7 +921,7 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li } else { write_uint8(s->s, TAG_INT64); - write_int64(s->s, *(int64_t*)data); + write_uint64(s->s, *(int64_t*)data); } } else if (jl_typeis(v, jl_int32_type)) { @@ -1480,7 +1475,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp) ios_seek(s, initial_pos); write_uint64(s, pos - initial_pos); ios_seek(s, pos); - write_int64(s, 0); + write_uint64(s, 0); } return pos; } @@ -2715,7 +2710,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) // Go back and update the source-text position to point to the current position int64_t posfile = ios_pos(&f); ios_seek(&f, srctextpos); - write_int64(&f, posfile); + write_uint64(&f, posfile); ios_seek_end(&f); // Each source-text file is written as // int32: length of abspath diff --git a/src/ircode.c b/src/ircode.c index 46056114a1c8d..34428cff79fe8 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -80,7 +80,7 @@ static void jl_encode_as_indexed_root(jl_ircode_state *s, jl_value_t *v) assert(id >= 0); if (rr.key) { write_uint8(s->s, TAG_RELOC_METHODROOT); - write_int64(s->s, rr.key); + write_uint64(s->s, rr.key); } if (id < 256) { write_uint8(s->s, TAG_METHODROOT); @@ -283,7 +283,7 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) } else { write_uint8(s->s, TAG_INT64); - write_int64(s->s, *(int64_t*)data); + write_uint64(s->s, *(int64_t*)data); } } else if (jl_typeis(v, jl_int32_type)) { diff --git a/src/serialize.h b/src/serialize.h index 564600974d1d6..020cafc74c962 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -92,7 +92,7 @@ static inline uint64_t read_uint64(ios_t *s) JL_NOTSAFEPOINT return x; } -static void write_int64(ios_t *s, int64_t i) JL_NOTSAFEPOINT +static inline void write_uint64(ios_t *s, uint64_t i) JL_NOTSAFEPOINT { ios_write(s, (char*)&i, 8); } @@ -121,6 +121,19 @@ static inline uint32_t read_uint32(ios_t *s) JL_NOTSAFEPOINT return x; } +#ifdef _P64 +#define write_uint(s, i) write_uint64(s, i) +#else +#define write_uint(s, i) write_uint32(s, i) +#endif + +#ifdef _P64 +#define read_uint(s) read_uint64(s) +#else +#define read_uint(s) read_uint32(s) +#endif + + void *jl_lookup_ser_tag(jl_value_t *v); void *jl_lookup_common_symbol(jl_value_t *v); jl_value_t *jl_deser_tag(uint8_t tag); diff --git a/src/staticdata.c b/src/staticdata.c index 827afac81b6da..aa8b3944eb14a 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -358,9 +358,28 @@ typedef enum { } jl_callingconv_t; +//#ifdef _P64 +//#define RELOC_TAG_OFFSET 61 +//#else // this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data. -// if a larger size is required, will need to add support for writing larger relocations in many cases below #define RELOC_TAG_OFFSET 29 +//#endif + +#if RELOC_TAG_OFFSET <= 32 +typedef uint32_t reloc_t; +#else +typedef uint64_t reloc_t; +#endif +static void write_reloc_t(ios_t *s, uintptr_t reloc_id) JL_NOTSAFEPOINT +{ + if (sizeof(reloc_t) <= sizeof(uint32_t)) { + assert(reloc_id < UINT32_MAX); + write_uint32(s, reloc_id); + } + else { + write_uint64(s, reloc_id); + } +} // --- Static Compile --- @@ -621,10 +640,9 @@ static void record_gvar(jl_serializer_state *s, int gid, uintptr_t reloc_id) JL_ { if (gid == 0) return; - ios_ensureroom(s->gvar_record, gid * sizeof(uint32_t)); - ios_seek(s->gvar_record, (gid - 1) * sizeof(uint32_t)); - assert(reloc_id < UINT32_MAX); - write_uint32(s->gvar_record, reloc_id); + ios_ensureroom(s->gvar_record, gid * sizeof(reloc_t)); + ios_seek(s->gvar_record, (gid - 1) * sizeof(reloc_t)); + write_reloc_t(s->gvar_record, reloc_id); } @@ -643,7 +661,7 @@ static void write_padding(ios_t *s, size_t nb) JL_NOTSAFEPOINT static void write_pointer(ios_t *s) JL_NOTSAFEPOINT { assert((ios_pos(s) & (sizeof(void*) - 1)) == 0 && "stream misaligned for writing a word-sized value"); - write_padding(s, sizeof(void*)); + write_uint(s, 0); } // Return the integer `id` for `v`. Generically this is looked up in `backref_table`, @@ -1098,18 +1116,20 @@ static void jl_write_values(jl_serializer_state *s) assert(invokeptr_id > 0); ios_ensureroom(s->fptr_record, invokeptr_id * sizeof(void*)); ios_seek(s->fptr_record, (invokeptr_id - 1) * sizeof(void*)); - write_uint32(s->fptr_record, ~reloc_offset); + write_reloc_t(s->fptr_record, (uint32_t)~reloc_offset); #ifdef _P64 - write_padding(s->fptr_record, 4); + if (sizeof(reloc_t) < 8) + write_padding(s->fptr_record, 8 - sizeof(reloc_t)); #endif } if (specfptr_id) { assert(specfptr_id > invokeptr_id && specfptr_id > 0); ios_ensureroom(s->fptr_record, specfptr_id * sizeof(void*)); ios_seek(s->fptr_record, (specfptr_id - 1) * sizeof(void*)); - write_uint32(s->fptr_record, reloc_offset); + write_reloc_t(s->fptr_record, reloc_offset); #ifdef _P64 - write_padding(s->fptr_record, 4); + if (sizeof(reloc_t) < 8) + write_padding(s->fptr_record, 8 - sizeof(reloc_t)); #endif } } @@ -1232,14 +1252,6 @@ static void jl_write_gv_tagrefs(jl_serializer_state *s) } } -static inline uint32_t load_uint32(uintptr_t *base) -{ - uint32_t v = jl_load_unaligned_i32((void*)*base); - *base += 4; - return v; -} - - // In deserialization, create Symbols and set up the // index for backreferencing static void jl_read_symbols(jl_serializer_state *s) @@ -1248,7 +1260,8 @@ static void jl_read_symbols(jl_serializer_state *s) uintptr_t base = (uintptr_t)&s->symbols->buf[0]; uintptr_t end = base + s->symbols->size; while (base < end) { - uint32_t len = load_uint32(&base); + uint32_t len = jl_load_unaligned_i32((void*)base); + base += 4; const char *str = (const char*)base; base += len + 1; //printf("symbol %3d: %s\n", len, str); @@ -1306,7 +1319,7 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) } // Compute target location at deserialization -static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uint32_t reloc_id) +static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uintptr_t reloc_id) { enum RefTags tag = (enum RefTags)(reloc_id >> RELOC_TAG_OFFSET); size_t offset = (reloc_id & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1)); @@ -1378,10 +1391,9 @@ static void jl_write_skiplist(ios_t *s, char *base, size_t size, arraylist_t *li *pv = get_reloc_for_item(item, *pv); // record pos in relocations list // TODO: save space by using delta-compression - assert(pos < UINT32_MAX); - write_uint32(s, pos); + write_reloc_t(s, pos); } - write_uint32(s, 0); + write_reloc_t(s, 0); } @@ -1398,9 +1410,8 @@ static void jl_read_relocations(jl_serializer_state *s, uint8_t bits) uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; while (1) { - uintptr_t val = (uintptr_t)&s->relocs->buf[s->relocs->bpos]; - uint32_t offset = load_uint32(&val); - s->relocs->bpos += sizeof(uint32_t); + uintptr_t offset = *(reloc_t*)&s->relocs->buf[(uintptr_t)s->relocs->bpos]; + s->relocs->bpos += sizeof(reloc_t); if (offset == 0) break; uintptr_t *pv = (uintptr_t*)(base + offset); @@ -1410,16 +1421,17 @@ static void jl_read_relocations(jl_serializer_state *s, uint8_t bits) } } -static char* sysimg_base; -static char* sysimg_relocs; +static char *sysimg_base; +static char *sysimg_relocs; void gc_sweep_sysimg(void) { - uintptr_t base = (uintptr_t)sysimg_base; - uintptr_t relocs = (uintptr_t)sysimg_relocs; - if (relocs == 0) + char *base = sysimg_base; + reloc_t *relocs = (reloc_t*)sysimg_relocs; + if (relocs == NULL) return; while (1) { - uint32_t offset = load_uint32(&relocs); + uintptr_t offset = *relocs; + relocs++; if (offset == 0) break; jl_taggedvalue_t *o = (jl_taggedvalue_t*)(base + offset); @@ -1431,13 +1443,12 @@ void gc_sweep_sysimg(void) static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) { if (v == NULL) { - write_uint32(s->s, 0); + write_reloc_t(s->s, 0); return; } uintptr_t item = backref_id(s, v); uintptr_t reloc = get_reloc_for_item(item, 0); - assert(reloc < UINT32_MAX); - write_uint32(s->s, reloc); + write_reloc_t(s->s, reloc); } @@ -1445,9 +1456,8 @@ static jl_value_t *jl_read_value(jl_serializer_state *s) { uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; - uintptr_t val = base + s->s->bpos; - uint32_t offset = load_uint32(&val); - s->s->bpos += sizeof(uint32_t); + uintptr_t offset = *(reloc_t*)(base + (uintptr_t)s->s->bpos); + s->s->bpos += sizeof(reloc_t); if (offset == 0) return NULL; return (jl_value_t*)get_item_for_reloc(s, base, size, offset); @@ -1470,12 +1480,11 @@ static void jl_update_all_fptrs(jl_serializer_state *s) jl_method_instance_t **linfos = (jl_method_instance_t**)&s->fptr_record->buf[0]; uint32_t clone_idx = 0; for (i = 0; i < sysimg_fvars_max; i++) { - uintptr_t val = (uintptr_t)&linfos[i]; - uint32_t offset = load_uint32(&val); + reloc_t offset = *(reloc_t*)&linfos[i]; linfos[i] = NULL; if (offset != 0) { int specfunc = 1; - if (offset & ((uintptr_t)1 << (8 * sizeof(uint32_t) - 1))) { + if (offset & ((uintptr_t)1 << (8 * sizeof(reloc_t) - 1))) { // if high bit is set, this is the func wrapper, not the specfunc specfunc = 0; offset = ~offset; @@ -1517,15 +1526,16 @@ static void jl_update_all_gvars(jl_serializer_state *s) size_t gvname_index = 0; uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; - uintptr_t gvars = (uintptr_t)&s->gvar_record->buf[0]; - uintptr_t end = gvars + s->gvar_record->size; + reloc_t *gvars = (reloc_t*)&s->gvar_record->buf[0]; + reloc_t *end = gvars + s->gvar_record->size / sizeof(reloc_t); while (gvars < end) { - uint32_t offset = load_uint32(&gvars); + uintptr_t offset = *gvars; if (offset) { uintptr_t v = get_item_for_reloc(s, base, size, offset); *sysimg_gvars(sysimg_gvars_base, gvname_index) = v; } gvname_index += 1; + gvars++; } } @@ -1541,14 +1551,14 @@ static void jl_finalize_serializer(jl_serializer_state *s, arraylist_t *list) size_t item = (size_t)list->items[i]; size_t reloc_offset = (size_t)layout_table.items[item]; assert(reloc_offset != 0); - write_uint32(s->s, (uint32_t)reloc_offset); - write_uint32(s->s, (uint32_t)((uintptr_t)list->items[i + 1])); + write_reloc_t(s->s, reloc_offset); + write_uint8(s->s, (uintptr_t)list->items[i + 1]); } - write_uint32(s->s, 0); + write_reloc_t(s->s, 0); } -static void jl_reinit_item(jl_value_t *v, int how) JL_GC_DISABLED +static void jl_reinit_item(jl_value_t *v, uint8_t how) JL_GC_DISABLED { switch (how) { case 1: { // rehash IdDict @@ -1600,11 +1610,17 @@ static void jl_finalize_deserializer(jl_serializer_state *s) JL_GC_DISABLED // run reinitialization functions uintptr_t base = (uintptr_t)&s->s->buf[0]; while (1) { - size_t offset = read_uint32(s->s); + size_t offset; + if (sizeof(reloc_t) <= 4) { + offset = read_uint32(s->s); + } + else { + offset = read_uint64(s->s); + } if (offset == 0) break; jl_value_t *v = (jl_value_t*)(base + offset); - jl_reinit_item(v, read_uint32(s->s)); + jl_reinit_item(v, read_uint8(s->s)); } } @@ -1932,7 +1948,7 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED } { // step 2: build all the sysimg sections - write_padding(&sysimg, sizeof(uint32_t)); + write_padding(&sysimg, sizeof(uintptr_t)); jl_write_values(&s); jl_write_relocations(&s); jl_write_gv_syms(&s, jl_get_root_symbol()); @@ -1948,7 +1964,7 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED ); jl_exit(1); } - if (const_data.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)*sizeof(void*)) { + if (const_data.size / sizeof(void*) > ((uintptr_t)1 << RELOC_TAG_OFFSET)) { jl_printf( JL_STDERR, "ERROR: system image too large: const_data.size is %jd but the limit is %" PRIxPTR "\n", @@ -1959,40 +1975,45 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED } // step 3: combine all of the sections into one file - write_uint32(f, sysimg.size - sizeof(uint32_t)); - ios_seek(&sysimg, sizeof(uint32_t)); + write_uint(f, sysimg.size - sizeof(uintptr_t)); + ios_seek(&sysimg, sizeof(uintptr_t)); ios_copyall(f, &sysimg); ios_close(&sysimg); - write_uint32(f, const_data.size); + write_uint(f, const_data.size); // realign stream to max-alignment for data - write_padding(f, LLT_ALIGN(ios_pos(f), 16) - ios_pos(f)); + write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); ios_seek(&const_data, 0); ios_copyall(f, &const_data); ios_close(&const_data); - write_uint32(f, symbols.size); + write_uint(f, symbols.size); + write_padding(f, LLT_ALIGN(ios_pos(f), 8) - ios_pos(f)); ios_seek(&symbols, 0); ios_copyall(f, &symbols); ios_close(&symbols); - write_uint32(f, relocs.size); + write_uint(f, relocs.size); + write_padding(f, LLT_ALIGN(ios_pos(f), 8) - ios_pos(f)); ios_seek(&relocs, 0); ios_copyall(f, &relocs); ios_close(&relocs); - write_uint32(f, gvar_record.size); + write_uint(f, gvar_record.size); + write_padding(f, LLT_ALIGN(ios_pos(f), 8) - ios_pos(f)); ios_seek(&gvar_record, 0); ios_copyall(f, &gvar_record); ios_close(&gvar_record); - write_uint32(f, fptr_record.size); + write_uint(f, fptr_record.size); + write_padding(f, LLT_ALIGN(ios_pos(f), 8) - ios_pos(f)); ios_seek(&fptr_record, 0); ios_copyall(f, &fptr_record); ios_close(&fptr_record); { // step 4: record locations of special roots s.s = f; + write_padding(f, LLT_ALIGN(ios_pos(f), 8) - ios_pos(f)); size_t i; for (i = 0; tags[i] != NULL; i++) { jl_value_t *tag = *tags[i]; @@ -2001,8 +2022,8 @@ static void jl_save_system_image_to_stream(ios_t *f) JL_GC_DISABLED jl_write_value(&s, jl_global_roots_table); jl_write_value(&s, s.ptls->root_task->tls); write_uint32(f, jl_get_gs_ctr()); - write_uint32(f, jl_atomic_load_acquire(&jl_world_counter)); - write_uint32(f, jl_typeinf_world); + write_uint(f, jl_atomic_load_acquire(&jl_world_counter)); + write_uint(f, jl_typeinf_world); jl_finalize_serializer(&s, &reinit_list); jl_finalize_serializer(&s, &ccallable_list); } @@ -2089,37 +2110,43 @@ static void jl_restore_system_image_from_stream(ios_t *f) JL_GC_DISABLED // step 1: read section map assert(ios_pos(f) == 0 && f->bm == bm_mem); - size_t sizeof_sysimg = read_uint32(f); - ios_static_buffer(&sysimg, f->buf, sizeof_sysimg + sizeof(uint32_t)); + size_t sizeof_sysimg = read_uint(f); + ios_static_buffer(&sysimg, f->buf, sizeof_sysimg + sizeof(uintptr_t)); ios_skip(f, sizeof_sysimg); - size_t sizeof_constdata = read_uint32(f); + size_t sizeof_constdata = read_uint(f); // realign stream to max-alignment for data - ios_seek(f, LLT_ALIGN(ios_pos(f), 16)); + ios_seek(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT)); ios_static_buffer(&const_data, f->buf + f->bpos, sizeof_constdata); ios_skip(f, sizeof_constdata); - size_t sizeof_symbols = read_uint32(f); + size_t sizeof_symbols = read_uint(f); + ios_seek(f, LLT_ALIGN(ios_pos(f), 8)); ios_static_buffer(&symbols, f->buf + f->bpos, sizeof_symbols); ios_skip(f, sizeof_symbols); - size_t sizeof_relocations = read_uint32(f); + size_t sizeof_relocations = read_uint(f); + ios_seek(f, LLT_ALIGN(ios_pos(f), 8)); assert(!ios_eof(f)); ios_static_buffer(&relocs, f->buf + f->bpos, sizeof_relocations); ios_skip(f, sizeof_relocations); - size_t sizeof_gvar_record = read_uint32(f); + size_t sizeof_gvar_record = read_uint(f); + ios_seek(f, LLT_ALIGN(ios_pos(f), 8)); assert(!ios_eof(f)); ios_static_buffer(&gvar_record, f->buf + f->bpos, sizeof_gvar_record); ios_skip(f, sizeof_gvar_record); - size_t sizeof_fptr_record = read_uint32(f); + size_t sizeof_fptr_record = read_uint(f); + ios_seek(f, LLT_ALIGN(ios_pos(f), 8)); assert(!ios_eof(f)); ios_static_buffer(&fptr_record, f->buf + f->bpos, sizeof_fptr_record); ios_skip(f, sizeof_fptr_record); // step 2: get references to special values s.s = f; + ios_seek(f, LLT_ALIGN(ios_pos(f), 8)); + assert(!ios_eof(f)); size_t i; for (i = 0; tags[i] != NULL; i++) { jl_value_t **tag = tags[i]; @@ -2135,8 +2162,8 @@ static void jl_restore_system_image_from_stream(ios_t *f) JL_GC_DISABLED jl_init_box_caches(); uint32_t gs_ctr = read_uint32(f); - jl_atomic_store_release(&jl_world_counter, read_uint32(f)); - jl_typeinf_world = read_uint32(f); + jl_atomic_store_release(&jl_world_counter, read_uint(f)); + jl_typeinf_world = read_uint(f); jl_set_gs_ctr(gs_ctr); s.s = NULL; From 40c37784dea8ad2b1f3f16c564e92cf5762044f1 Mon Sep 17 00:00:00 2001 From: Sukera <11753998+Seelengrab@users.noreply.github.com> Date: Sun, 2 Oct 2022 09:18:27 +0200 Subject: [PATCH 28/41] Add compat note for `@testset foo()` (#46980) Co-authored-by: Sukera (cherry picked from commit 45fd204b9fa7064eba75acaec5f367f466086512) --- stdlib/Test/src/Test.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 95cd1ecccd9c3..b9be89826cd20 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1268,6 +1268,9 @@ when they all pass (the default is `false`). - `showtiming`: if `true`, the duration of each displayed testset is shown (the default is `true`). +!!! compat "Julia 1.8" + `@testset foo()` requires at least Julia 1.8. + The description string accepts interpolation from the loop indices. If no description is provided, one is constructed based on the variables. If a function call is provided, its name will be used. Explicit description strings override this behavior. From 36a17adba539246f094665730f63fac5201c5829 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 26 Oct 2022 17:39:35 -0400 Subject: [PATCH 29/41] subtype: fix miscount of Tuple Vararg matching Fix #47246 (cherry picked from commit 17515cdf16d380c269d748a88e82693f87db4d4a) --- src/subtype.c | 2 +- test/subtype.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/subtype.c b/src/subtype.c index bb229d4b76bd0..5ef77f0010b4e 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -1000,7 +1000,7 @@ static int subtype_tuple_tail(jl_datatype_t *xd, jl_datatype_t *yd, int8_t R, jl { size_t lx = jl_nparams(xd); size_t ly = jl_nparams(yd); - size_t i = 0, j = 0, vx = 0, vy = 0, x_reps = 1; + size_t i = 0, j = 0, vx = 0, vy = 0, x_reps = 0; jl_value_t *lastx = NULL, *lasty = NULL; jl_value_t *xi = NULL, *yi = NULL; diff --git a/test/subtype.jl b/test/subtype.jl index 9545838bff141..c6d2dada81d25 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2038,3 +2038,5 @@ struct B46871{T, N} <: Ref{A46871{T, N, N}} end for T in (B46871{Int, N} where {N}, B46871{Int}) # intentional duplication @testintersect(T, Ref{<:AbstractArray{<:Real, 3}}, B46871{Int, 3}) end + +@test !(Tuple{Any, Any, Any} <: Tuple{Any, Vararg{T}} where T) From 7ff390027049c2c179deb6158f05f55c1623e7f8 Mon Sep 17 00:00:00 2001 From: William Moses Date: Tue, 25 Oct 2022 17:34:54 -0400 Subject: [PATCH 30/41] Fix GC assertion on array of derived pointers (#47299) * Fix GC assertion on array of derived pointers * Add test (cherry picked from commit d885fc780c216cc869bf3f12c2174ee3bf0572f8) --- src/llvm-late-gc-lowering.cpp | 2 +- test/llvmpasses/late-lower-gc.ll | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 9f716db3898d5..ab0ffff8944ac 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -419,7 +419,7 @@ unsigned getCompositeNumElements(Type *T) { // Walk through a Type, and record the element path to every tracked value inside void TrackCompositeType(Type *T, std::vector &Idxs, std::vector> &Numberings) { if (isa(T)) { - if (T->getPointerAddressSpace() == AddressSpace::Tracked) + if (isSpecialPtr(T)) Numberings.push_back(Idxs); } else if (isa(T) || isa(T) || isa(T)) { diff --git a/test/llvmpasses/late-lower-gc.ll b/test/llvmpasses/late-lower-gc.ll index 22cb558c54158..fb1961a3c230a 100644 --- a/test/llvmpasses/late-lower-gc.ll +++ b/test/llvmpasses/late-lower-gc.ll @@ -124,6 +124,25 @@ top: ; CHECK: ret i32 } +define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { + %v2 = call {}*** @julia.get_pgcstack() + %e0 = extractvalue [2 x {} addrspace(10)* addrspace(11)*] %ar, 0 + %l0 = load {} addrspace(10)*, {} addrspace(10)* addrspace(11)* %e0 + %e1 = extractvalue [2 x {} addrspace(10)* addrspace(11)*] %ar, 1 + %l1 = load {} addrspace(10)*, {} addrspace(10)* addrspace(11)* %e1 + %r = call i32 @callee_root({} addrspace(10)* %l0, {} addrspace(10)* %l1) + ret void +} + +; CHECK-LABEL: @decayar +; CHECK: %gcframe = call {} addrspace(10)** @julia.new_gc_frame(i32 2) +; CHECK: %1 = call {} addrspace(10)** @julia.get_gc_frame_slot({} addrspace(10)** %gcframe, i32 1) +; CHECK: store {} addrspace(10)* %l0, {} addrspace(10)** %1, align 8 +; CHECK: %2 = call {} addrspace(10)** @julia.get_gc_frame_slot({} addrspace(10)** %gcframe, i32 0) +; CHECK: store {} addrspace(10)* %l1, {} addrspace(10)** %2, align 8 +; CHECK: %r = call i32 @callee_root({} addrspace(10)* %l0, {} addrspace(10)* %l1) +; CHECK: call void @julia.pop_gc_frame({} addrspace(10)** %gcframe) + !0 = !{i64 0, i64 23} !1 = !{} !2 = distinct !{!2} From 7e07aa0132970727e063c1ede526b423d99b491f Mon Sep 17 00:00:00 2001 From: apaz Date: Tue, 18 Oct 2022 11:14:39 -0500 Subject: [PATCH 31/41] sysimg relocation lists now use LEB128 format (#47204) Appears to reduce the data size from about 165MB to 147MB! (cherry picked from commit 94736a4ddc7c63ad87774c5eb03cbc879154a3b1) --- src/staticdata.c | 101 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 71 insertions(+), 30 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index aa8b3944eb14a..30cec95bee71b 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -358,12 +358,12 @@ typedef enum { } jl_callingconv_t; -//#ifdef _P64 -//#define RELOC_TAG_OFFSET 61 -//#else +#ifdef _P64 +#define RELOC_TAG_OFFSET 61 +#else // this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data. #define RELOC_TAG_OFFSET 29 -//#endif +#endif #if RELOC_TAG_OFFSET <= 32 typedef uint32_t reloc_t; @@ -1116,7 +1116,7 @@ static void jl_write_values(jl_serializer_state *s) assert(invokeptr_id > 0); ios_ensureroom(s->fptr_record, invokeptr_id * sizeof(void*)); ios_seek(s->fptr_record, (invokeptr_id - 1) * sizeof(void*)); - write_reloc_t(s->fptr_record, (uint32_t)~reloc_offset); + write_reloc_t(s->fptr_record, (reloc_t)~reloc_offset); #ifdef _P64 if (sizeof(reloc_t) < 8) write_padding(s->fptr_record, 8 - sizeof(reloc_t)); @@ -1380,41 +1380,71 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas } -static void jl_write_skiplist(ios_t *s, char *base, size_t size, arraylist_t *list) +static void jl_write_reloclist(ios_t *s, char *base, size_t size, arraylist_t *list) { - size_t i; - for (i = 0; i < list->len; i += 2) { + for (size_t i = 0; i < list->len; i += 2) { + size_t last_pos = i ? (size_t)list->items[i - 2] : 0; size_t pos = (size_t)list->items[i]; size_t item = (size_t)list->items[i + 1]; uintptr_t *pv = (uintptr_t*)(base + pos); assert(pos < size && pos != 0); *pv = get_reloc_for_item(item, *pv); - // record pos in relocations list - // TODO: save space by using delta-compression - write_reloc_t(s, pos); + + // write pos as compressed difference. + size_t pos_diff = pos - last_pos; + while (pos_diff) { + assert(pos_diff >= 0); + if (pos_diff <= 127) { + write_int8(s, pos_diff); + break; + } + else { + // Extract the next 7 bits + int8_t ns = pos_diff & (int8_t)0x7F; + pos_diff >>= 7; + // Set the high bit if there's still more + ns |= (!!pos_diff) << 7; + write_int8(s, ns); + } + } } - write_reloc_t(s, 0); + write_int8(s, 0); } static void jl_write_relocations(jl_serializer_state *s) { char *base = &s->s->buf[0]; - jl_write_skiplist(s->relocs, base, s->s->size, &s->gctags_list); - jl_write_skiplist(s->relocs, base, s->s->size, &s->relocs_list); + jl_write_reloclist(s->relocs, base, s->s->size, &s->gctags_list); + jl_write_reloclist(s->relocs, base, s->s->size, &s->relocs_list); } - -static void jl_read_relocations(jl_serializer_state *s, uint8_t bits) +static void jl_read_reloclist(jl_serializer_state *s, uint8_t bits) { - uintptr_t base = (uintptr_t)&s->s->buf[0]; + uintptr_t base = (uintptr_t)s->s->buf; size_t size = s->s->size; + uintptr_t last_pos = 0; + uint8_t *current = (uint8_t *)(s->relocs->buf + s->relocs->bpos); while (1) { - uintptr_t offset = *(reloc_t*)&s->relocs->buf[(uintptr_t)s->relocs->bpos]; - s->relocs->bpos += sizeof(reloc_t); - if (offset == 0) + // Read the offset of the next object + size_t pos_diff = 0; + size_t cnt = 0; + while (1) { + assert(s->relocs->bpos <= s->relocs->size); + assert((char *)current <= (char *)(s->relocs->buf + s->relocs->size)); + int8_t c = *current++; + s->relocs->bpos += 1; + + pos_diff |= ((size_t)c & 0x7F) << (7 * cnt++); + if ((c >> 7) == 0) + break; + } + if (pos_diff == 0) break; - uintptr_t *pv = (uintptr_t*)(base + offset); + + uintptr_t pos = last_pos + pos_diff; + last_pos = pos; + uintptr_t *pv = (uintptr_t *)(base + pos); uintptr_t v = *pv; v = get_item_for_reloc(s, base, size, v); *pv = v | bits; @@ -1425,16 +1455,27 @@ static char *sysimg_base; static char *sysimg_relocs; void gc_sweep_sysimg(void) { - char *base = sysimg_base; - reloc_t *relocs = (reloc_t*)sysimg_relocs; - if (relocs == NULL) + if (!sysimg_relocs) return; + uintptr_t base = (uintptr_t)sysimg_base; + uintptr_t last_pos = 0; + uint8_t *current = (uint8_t *)sysimg_relocs; while (1) { - uintptr_t offset = *relocs; - relocs++; - if (offset == 0) + // Read the offset of the next object + size_t pos_diff = 0; + size_t cnt = 0; + while (1) { + int8_t c = *current++; + pos_diff |= ((size_t)c & 0x7F) << (7 * cnt++); + if ((c >> 7) == 0) + break; + } + if (pos_diff == 0) break; - jl_taggedvalue_t *o = (jl_taggedvalue_t*)(base + offset); + + uintptr_t pos = last_pos + pos_diff; + last_pos = pos; + jl_taggedvalue_t *o = (jl_taggedvalue_t *)(base + pos); o->bits.gc = GC_OLD; } } @@ -2177,10 +2218,10 @@ static void jl_restore_system_image_from_stream(ios_t *f) JL_GC_DISABLED jl_gc_set_permalloc_region((void*)sysimg_base, (void*)(sysimg_base + sysimg.size)); s.s = &sysimg; - jl_read_relocations(&s, GC_OLD_MARKED); // gctags + jl_read_reloclist(&s, GC_OLD_MARKED); // gctags size_t sizeof_tags = ios_pos(&relocs); (void)sizeof_tags; - jl_read_relocations(&s, 0); // general relocs + jl_read_reloclist(&s, 0); // general relocs ios_close(&relocs); ios_close(&const_data); jl_update_all_gvars(&s); // gvars relocs From 6caa01342c60afbf78233a3da523a8caffd67948 Mon Sep 17 00:00:00 2001 From: Jeffrey Sarnoff Date: Mon, 3 Oct 2022 11:38:35 -0400 Subject: [PATCH 32/41] bugfix: fld1 order of ops (#46938) * bugfix: fld1 order of ops fixes https://github.com/JuliaLang/julia/issues/28973 (cherry picked from commit fcdc5bc21296520c04349c901e234477c5e2ffad) --- NEWS.md | 1 + base/operators.jl | 2 +- test/operators.jl | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 91fb4bf6d3b5f..62683964eb884 100644 --- a/NEWS.md +++ b/NEWS.md @@ -111,6 +111,7 @@ New library features * `extrema` now accepts an `init` keyword argument ([#36265], [#43604]). * `Iterators.countfrom` now accepts any type that defines `+` ([#37747]). * `@time` now separates out % time spent recompiling invalidated methods ([#45015]). +* An issue with order of operations in `fld1` is now fixed ([#28973]). Standard library changes ------------------------ diff --git a/base/operators.jl b/base/operators.jl index e42699062f016..e8e31ec47cf16 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -855,7 +855,7 @@ julia> x == (fld1(x, y) - 1) * y + mod1(x, y) true ``` """ -fld1(x::T, y::T) where {T<:Real} = (m = mod1(x, y); fld(x + y - m, y)) +fld1(x::T, y::T) where {T<:Real} = (m = mod1(x, y); fld((x - m) + y, y)) function fld1(x::T, y::T) where T<:Integer d = div(x, y) return d + (!signbit(x ⊻ y) & (d * y != x)) diff --git a/test/operators.jl b/test/operators.jl index 1c5d2d33bf0f8..432456a2a3b42 100644 --- a/test/operators.jl +++ b/test/operators.jl @@ -261,6 +261,9 @@ end end @test fldmod1(4.0, 3) == fldmod1(4, 3) + + # issue 28973 + @test fld1(0.4, 0.9) == fld1(nextfloat(0.4), 0.9) == 1.0 end @testset "Fix12" begin From e78a9d2896d26322fce1c521fd2e25dd5778b954 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 27 Oct 2022 16:16:47 -0400 Subject: [PATCH 33/41] fix `--compile=all` option, and some improvements for `--strip-ir` (#46935) (#47357) (cherry picked from commit 6443a468817cba670e545c09561c16f2c30f31b2) (cherry picked from commit 3562f035f7351e4d7de8ecb37cc0de1044f781df) --- base/compiler/utilities.jl | 5 +- src/gf.c | 4 +- src/julia_internal.h | 2 +- src/precompile.c | 101 +++++++++++++------------------------ src/toplevel.c | 32 ++++++------ 5 files changed, 56 insertions(+), 88 deletions(-) diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index f7ff32ecbaefe..497c9c0252d17 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -127,7 +127,10 @@ function retrieve_code_info(linfo::MethodInstance) end if c === nothing && isdefined(m, :source) src = m.source - if isa(src, Array{UInt8,1}) + if src === nothing + # can happen in images built with --strip-ir + return nothing + elseif isa(src, Array{UInt8,1}) c = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, src) else c = copy(src::CodeInfo) diff --git a/src/gf.c b/src/gf.c index ad84c1c523be8..f0dcdbb298528 100644 --- a/src/gf.c +++ b/src/gf.c @@ -281,8 +281,6 @@ jl_code_info_t *jl_type_infer(jl_method_instance_t *mi, size_t world, int force) JL_TIMING(INFERENCE); if (jl_typeinf_func == NULL) return NULL; - if (jl_is_method(mi->def.method) && mi->def.method->unspecialized == mi) - return NULL; // avoid inferring the unspecialized method static int in_inference; if (in_inference > 2) return NULL; @@ -2062,7 +2060,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t if (compile_option == JL_OPTIONS_COMPILE_OFF || compile_option == JL_OPTIONS_COMPILE_MIN) { jl_code_info_t *src = jl_code_for_interpreter(mi); - if (!jl_code_requires_compiler(src)) { + if (!jl_code_requires_compiler(src, 0)) { jl_code_instance_t *codeinst = jl_new_codeinst(mi, (jl_value_t*)jl_any_type, NULL, NULL, 0, 1, ~(size_t)0, 0, 0, jl_nothing, 0); diff --git a/src/julia_internal.h b/src/julia_internal.h index a7e9c0af2ad3d..0906782640406 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -530,7 +530,7 @@ jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method JL_PROPA JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types); jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *lam JL_PROPAGATES_ROOT); -int jl_code_requires_compiler(jl_code_info_t *src); +int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile); jl_code_info_t *jl_new_code_info_from_ir(jl_expr_t *ast); JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, diff --git a/src/precompile.c b/src/precompile.c index 5a43dc45f094e..d938cbe06bf25 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -113,13 +113,6 @@ void jl_write_compiler_output(void) // and expanding the Union may give a leaf function static void _compile_all_tvar_union(jl_value_t *methsig) { - if (!jl_is_unionall(methsig) && jl_is_dispatch_tupletype(methsig)) { - // usually can create a specialized version of the function, - // if the signature is already a dispatch type - if (jl_compile_hint((jl_tupletype_t*)methsig)) - return; - } - int tvarslen = jl_subtype_env_size(methsig); jl_value_t *sigbody = methsig; jl_value_t **roots; @@ -246,74 +239,49 @@ static void _compile_all_union(jl_value_t *sig) JL_GC_POP(); } -static void _compile_all_deq(jl_array_t *found) -{ - int found_i, found_l = jl_array_len(found); - jl_printf(JL_STDERR, "found %d uncompiled methods for compile-all\n", (int)found_l); - jl_method_instance_t *mi = NULL; - jl_value_t *src = NULL; - JL_GC_PUSH2(&mi, &src); - for (found_i = 0; found_i < found_l; found_i++) { - if (found_i % (1 + found_l / 300) == 0 || found_i == found_l - 1) // show 300 progress steps, to show progress without overwhelming log files - jl_printf(JL_STDERR, " %d / %d\r", found_i + 1, found_l); - jl_typemap_entry_t *ml = (jl_typemap_entry_t*)jl_array_ptr_ref(found, found_i); - jl_method_t *m = ml->func.method; - if (m->source == NULL) // TODO: generic implementations of generated functions - continue; - mi = jl_get_unspecialized(mi); - assert(mi == m->unspecialized); // make sure we didn't get tricked by a generated function, since we can't handle those - jl_code_instance_t *ucache = jl_get_method_inferred(mi, (jl_value_t*)jl_any_type, 1, ~(size_t)0); - if (ucache->invoke != NULL) - continue; - src = m->source; - assert(src); - // TODO: we could now enable storing inferred function pointers in the `unspecialized` cache - //src = jl_type_infer(mi, jl_atomic_load_acquire(&jl_world_counter), 1); - //if (ucache->invoke != NULL) - // continue; - - // first try to create leaf signatures from the signature declaration and compile those - _compile_all_union((jl_value_t*)ml->sig); - // then also compile the generic fallback - jl_generate_fptr_for_unspecialized(ucache); - } - JL_GC_POP(); - jl_printf(JL_STDERR, "\n"); -} - -static int compile_all_enq__(jl_typemap_entry_t *ml, void *env) +static int compile_all_collect__(jl_typemap_entry_t *ml, void *env) { - jl_array_t *found = (jl_array_t*)env; - // method definition -- compile template field + jl_array_t *allmeths = (jl_array_t*)env; jl_method_t *m = ml->func.method; if (m->source) { - // found a method to compile - jl_array_ptr_1d_push(found, (jl_value_t*)ml); + // method has a non-generated definition; can be compiled generically + jl_array_ptr_1d_push(allmeths, (jl_value_t*)m); } return 1; } - -static int compile_all_enq_(jl_methtable_t *mt, void *env) +static int compile_all_collect_(jl_methtable_t *mt, void *env) { - jl_typemap_visitor(mt->defs, compile_all_enq__, env); + jl_typemap_visitor(mt->defs, compile_all_collect__, env); return 1; } -static void jl_compile_all_defs(void) +static void jl_compile_all_defs(jl_array_t *mis) { - // this "found" array will contain - // TypeMapEntries for Methods and MethodInstances that need to be compiled - jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); - while (1) { - jl_foreach_reachable_mtable(compile_all_enq_, m); - size_t changes = jl_array_len(m); - if (!changes) - break; - _compile_all_deq(m); - jl_array_del_end(m, changes); + jl_array_t *allmeths = jl_alloc_vec_any(0); + JL_GC_PUSH1(&allmeths); + + jl_foreach_reachable_mtable(compile_all_collect_, allmeths); + + size_t i, l = jl_array_len(allmeths); + for (i = 0; i < l; i++) { + jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i); + if (jl_isa_compileable_sig((jl_tupletype_t*)m->sig, m)) { + // method has a single compileable specialization, e.g. its definition + // signature is concrete. in this case we can just hint it. + jl_compile_hint((jl_tupletype_t*)m->sig); + } + else { + // first try to create leaf signatures from the signature declaration and compile those + _compile_all_union(m->sig); + + // finally, compile a fully generic fallback that can work for all arguments + jl_method_instance_t *unspec = jl_get_unspecialized(m); + if (unspec) + jl_array_ptr_1d_push(mis, (jl_value_t*)unspec); + } } + JL_GC_POP(); } @@ -371,14 +339,13 @@ static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) static void *jl_precompile(int all) { - if (all) - jl_compile_all_defs(); - // this "found" array will contain function - // type signatures that were inferred but haven't been compiled + // array of MethodInstances and ccallable aliases to include in the output jl_array_t *m = jl_alloc_vec_any(0); jl_array_t *m2 = NULL; jl_method_instance_t *mi = NULL; JL_GC_PUSH3(&m, &m2, &mi); + if (all) + jl_compile_all_defs(m); jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); m2 = jl_alloc_vec_any(0); for (size_t i = 0; i < jl_array_len(m); i++) { @@ -387,7 +354,7 @@ static void *jl_precompile(int all) mi = (jl_method_instance_t*)item; size_t min_world = 0; size_t max_world = ~(size_t)0; - if (!jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->def.method)) + if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->def.method)) mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); if (mi) jl_array_ptr_1d_push(m2, (jl_value_t*)mi); diff --git a/src/toplevel.c b/src/toplevel.c index 1f60a1b57c19c..5d703e3d183bb 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -337,7 +337,7 @@ JL_DLLEXPORT jl_module_t *jl_base_relative_to(jl_module_t *m) return jl_top_module; } -static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, int *has_opaque) +static void expr_attributes(jl_value_t *v, int *has_ccall, int *has_defs, int *has_opaque) { if (!jl_is_expr(v)) return; @@ -361,11 +361,11 @@ static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, i *has_defs = 1; } else if (head == jl_cfunction_sym) { - *has_intrinsics = 1; + *has_ccall = 1; return; } else if (head == jl_foreigncall_sym) { - *has_intrinsics = 1; + *has_ccall = 1; return; } else if (head == jl_new_opaque_closure_sym) { @@ -389,7 +389,7 @@ static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, i } if (called) { if (jl_is_intrinsic(called) && jl_unbox_int32(called) == (int)llvmcall) { - *has_intrinsics = 1; + *has_ccall = 1; } if (called == jl_builtin__typebody) { *has_defs = 1; @@ -401,28 +401,28 @@ static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, i for (i = 0; i < jl_array_len(e->args); i++) { jl_value_t *a = jl_exprarg(e, i); if (jl_is_expr(a)) - expr_attributes(a, has_intrinsics, has_defs, has_opaque); + expr_attributes(a, has_ccall, has_defs, has_opaque); } } -int jl_code_requires_compiler(jl_code_info_t *src) +int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile) { jl_array_t *body = src->code; assert(jl_typeis(body, jl_array_any_type)); size_t i; - int has_intrinsics = 0, has_defs = 0, has_opaque = 0; - if (jl_has_meta(body, jl_force_compile_sym)) + int has_ccall = 0, has_defs = 0, has_opaque = 0; + if (include_force_compile && jl_has_meta(body, jl_force_compile_sym)) return 1; for(i=0; i < jl_array_len(body); i++) { jl_value_t *stmt = jl_array_ptr_ref(body,i); - expr_attributes(stmt, &has_intrinsics, &has_defs, &has_opaque); - if (has_intrinsics) + expr_attributes(stmt, &has_ccall, &has_defs, &has_opaque); + if (has_ccall) return 1; } return 0; } -static void body_attributes(jl_array_t *body, int *has_intrinsics, int *has_defs, int *has_loops, int *has_opaque, int *forced_compile) +static void body_attributes(jl_array_t *body, int *has_ccall, int *has_defs, int *has_loops, int *has_opaque, int *forced_compile) { size_t i; *has_loops = 0; @@ -438,7 +438,7 @@ static void body_attributes(jl_array_t *body, int *has_intrinsics, int *has_defs *has_loops = 1; } } - expr_attributes(stmt, has_intrinsics, has_defs, has_opaque); + expr_attributes(stmt, has_ccall, has_defs, has_opaque); } *forced_compile = jl_has_meta(body, jl_force_compile_sym); } @@ -868,16 +868,16 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int return (jl_value_t*)ex; } - int has_intrinsics = 0, has_defs = 0, has_loops = 0, has_opaque = 0, forced_compile = 0; + int has_ccall = 0, has_defs = 0, has_loops = 0, has_opaque = 0, forced_compile = 0; assert(head == jl_thunk_sym); thk = (jl_code_info_t*)jl_exprarg(ex, 0); assert(jl_is_code_info(thk)); assert(jl_typeis(thk->code, jl_array_any_type)); - body_attributes((jl_array_t*)thk->code, &has_intrinsics, &has_defs, &has_loops, &has_opaque, &forced_compile); + body_attributes((jl_array_t*)thk->code, &has_ccall, &has_defs, &has_loops, &has_opaque, &forced_compile); jl_value_t *result; - if (forced_compile || has_intrinsics || - (!has_defs && fast && has_loops && + if (has_ccall || + ((forced_compile || (!has_defs && fast && has_loops)) && jl_options.compile_enabled != JL_OPTIONS_COMPILE_OFF && jl_options.compile_enabled != JL_OPTIONS_COMPILE_MIN && jl_get_module_compile(m) != JL_OPTIONS_COMPILE_OFF && From 47ef40710bd040d6df49e8a13099644705fd5ce6 Mon Sep 17 00:00:00 2001 From: Hendrik Ranocha Date: Mon, 3 Oct 2022 03:13:53 +0200 Subject: [PATCH 34/41] improve inferrability of `sort!` (#46921) Co-authored-by: Lilith Orion Hafner (cherry picked from commit 910506430dfa4f623974631b76270772c65cb8e4) --- base/sort.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/sort.jl b/base/sort.jl index b538428a8c424..276c0a448687b 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -914,7 +914,7 @@ function sortperm(v::AbstractVector; min, max = extrema(v) (diff, o1) = sub_with_overflow(max, min) (rangelen, o2) = add_with_overflow(diff, oneunit(diff)) - if !o1 && !o2 && rangelen < div(n,2) + if (!o1 && !o2)::Bool && rangelen < div(n,2) return sortperm_int_range(v, rangelen, min) end end From a719aff09ff6a8820485996279c974023277b62d Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Fri, 28 Oct 2022 08:36:13 -0400 Subject: [PATCH 35/41] Build system: when building the HTML docs, exclude SuiteSparse if `USE_GPL_LIBS=0` (#47361) --- doc/make.jl | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index 9cc463f5a84e4..7653f302ac495 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -24,6 +24,10 @@ cd(joinpath(@__DIR__, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) + if (dir == "SuiteSparse") && (!Base.USE_GPL_LIBS) + @info "Excluding SuiteSparse from the docs because Base.USE_GPL_LIBS is false" + continue + end sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") @@ -236,12 +240,14 @@ DocMeta.setdocmeta!( maybe_revise(:(using SparseArrays, LinearAlgebra)); recursive=true, warn=false, ) -DocMeta.setdocmeta!( - SuiteSparse, - :DocTestSetup, - maybe_revise(:(using SparseArrays, LinearAlgebra, SuiteSparse)); - recursive=true, warn=false, -) +if Base.USE_GPL_LIBS + DocMeta.setdocmeta!( + SuiteSparse, + :DocTestSetup, + maybe_revise(:(using SparseArrays, LinearAlgebra, SuiteSparse)); + recursive=true, warn=false, + ) +end DocMeta.setdocmeta!( UUIDs, :DocTestSetup, From 50ea71aa55128a50eb1f5e37625c65ca51876e7b Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 3 Nov 2022 22:41:46 +0900 Subject: [PATCH 36/41] fix up `Conditional`-forwarding for vararg methods (#47438) --- base/compiler/inferenceresult.jl | 2 +- test/compiler/inference.jl | 112 +++++++++++++++++-------------- 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/base/compiler/inferenceresult.jl b/base/compiler/inferenceresult.jl index 8e3d3d5917fd0..b806830a8c46b 100644 --- a/base/compiler/inferenceresult.jl +++ b/base/compiler/inferenceresult.jl @@ -71,7 +71,7 @@ function matching_cache_argtypes( # invalidate `Conditional` imposed on varargs if condargs !== nothing for (slotid, i) in condargs - if slotid ≥ last + if slotid ≥ last && (1 ≤ i ≤ length(isva_given_argtypes)) # `Conditional` is already widened to vararg-tuple otherwise isva_given_argtypes[i] = widenconditional(isva_given_argtypes[i]) end end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 00470aaccf248..298e467ba841e 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -2260,66 +2260,78 @@ function _g_ifelse_isa_() end @test Base.return_types(_g_ifelse_isa_, ()) == [Int] -@testset "Conditional forwarding" begin - # forward `Conditional` if it conveys a constraint on any other argument - ifelselike(cnd, x, y) = cnd ? x : y +# Conditional forwarding +# ====================== - @test Base.return_types((Any,Int,)) do x, y - ifelselike(isa(x, Int), x, y) - end |> only == Int - - # should work nicely with union-split - @test Base.return_types((Union{Int,Nothing},)) do x - ifelselike(isa(x, Int), x, 0) - end |> only == Int +# forward `Conditional` if it conveys a constraint on any other argument +ifelselike(cnd, x, y) = cnd ? x : y - @test Base.return_types((Any,Int)) do x, y - ifelselike(!isa(x, Int), y, x) - end |> only == Int +@test Base.return_types((Any,Int,)) do x, y + ifelselike(isa(x, Int), x, y) +end |> only == Int - @test Base.return_types((Any,Int)) do x, y - a = ifelselike(x === 0, x, 0) # ::Const(0) - if a == 0 - return y - else - return nothing # dead branch - end - end |> only == Int +# should work nicely with union-split +@test Base.return_types((Union{Int,Nothing},)) do x + ifelselike(isa(x, Int), x, 0) +end |> only == Int - # pick up the first if there are multiple constrained arguments - @test Base.return_types((Any,)) do x - ifelselike(isa(x, Int), x, x) - end |> only == Any +@test Base.return_types((Any,Int)) do x, y + ifelselike(!isa(x, Int), y, x) +end |> only == Int - # just propagate multiple constraints - ifelselike2(cnd1, cnd2, x, y, z) = cnd1 ? x : cnd2 ? y : z - @test Base.return_types((Any,Any)) do x, y - ifelselike2(isa(x, Int), isa(y, Int), x, y, 0) - end |> only == Int +@test Base.return_types((Any,Int)) do x, y + a = ifelselike(x === 0, x, 0) # ::Const(0) + if a == 0 + return y + else + return nothing # dead branch + end +end |> only == Int - # work with `invoke` - @test Base.return_types((Any,Any)) do x, y - Base.@invoke ifelselike(isa(x, Int), x, y::Int) - end |> only == Int +# pick up the first if there are multiple constrained arguments +@test Base.return_types((Any,)) do x + ifelselike(isa(x, Int), x, x) +end |> only == Any - # don't be confused with vararg method - vacond(cnd, va...) = cnd ? va : 0 - @test Base.return_types((Any,)) do x - # at runtime we will see `va::Tuple{Tuple{Int,Int}, Tuple{Int,Int}}` - vacond(isa(x, Tuple{Int,Int}), x, x) - end |> only == Union{Int,Tuple{Any,Any}} +# just propagate multiple constraints +ifelselike2(cnd1, cnd2, x, y, z) = cnd1 ? x : cnd2 ? y : z +@test Base.return_types((Any,Any)) do x, y + ifelselike2(isa(x, Int), isa(y, Int), x, y, 0) +end |> only == Int - # demonstrate extra constraint propagation for Base.ifelse - @test Base.return_types((Any,Int,)) do x, y - ifelse(isa(x, Int), x, y) - end |> only == Int +# work with `invoke` +@test Base.return_types((Any,Any)) do x, y + Base.@invoke ifelselike(isa(x, Int), x::Any, y::Int) +end |> only == Int - # slot as SSA - @test Base.return_types((Any,Vector{Any})) do x, y - z = x - ifelselike(isa(z, Int), z, length(y)) - end |> only === Int +# don't be confused with vararg method +vacond(cnd, va...) = cnd ? va : 0 +@test Base.return_types((Any,)) do x + # at runtime we will see `va::Tuple{Tuple{Int,Int}, Tuple{Int,Int}}` + vacond(isa(x, Tuple{Int,Int}), x, x) +end |> only == Union{Int,Tuple{Any,Any}} + +# https://github.com/JuliaLang/julia/issues/47435 +is_closed_ex(e::InvalidStateException) = true +is_closed_ex(e) = false +function issue47435() + try + catch e + println("caught $e: $(is_closed_ex(e))") + end end +@test only(Base.return_types(issue47435)) === Nothing + +# demonstrate extra constraint propagation for Base.ifelse +@test Base.return_types((Any,Int,)) do x, y + ifelse(isa(x, Int), x, y) +end |> only == Int + +# forward conditional information imposed on SSA that is alised to a slot +@test Base.return_types((Any,Vector{Any})) do x, y + z = x + ifelselike(isa(z, Int), z, length(y)) +end |> only === Int # Equivalence of Const(T.instance) and T for singleton types @test Const(nothing) ⊑ Nothing && Nothing ⊑ Const(nothing) From 10f616d9d4711b67e80ffefde2166a86c49a8807 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 6 Nov 2022 18:58:33 -0500 Subject: [PATCH 37/41] 1.8: Change the tracking branch for SuiteSparse from `master` to `release-julia-1.8`, but leave the commit (`f63732c`) unmodified --- stdlib/SuiteSparse.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/SuiteSparse.version b/stdlib/SuiteSparse.version index 27e835befbc38..9958a3cafa53b 100644 --- a/stdlib/SuiteSparse.version +++ b/stdlib/SuiteSparse.version @@ -1,4 +1,4 @@ -SUITESPARSE_BRANCH = master +SUITESPARSE_BRANCH = release-julia-1.8 SUITESPARSE_SHA1 = f63732c1c6adecb277d8f2981cc8c1883c321bcc SUITESPARSE_GIT_URL := https://github.com/JuliaSparse/SuiteSparse.jl.git SUITESPARSE_TAR_URL = https://api.github.com/repos/JuliaSparse/SuiteSparse.jl/tarball/$1 From f69b3612a5332bd70efc180ce5d8a658f289535d Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Mon, 7 Nov 2022 16:23:37 +0100 Subject: [PATCH 38/41] revert part of https://github.com/JuliaLang/julia/pull/44444/ which seems to make buildkite unhappy --- test/misc.jl | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/test/misc.jl b/test/misc.jl index bda0566650170..293b9d868c3b3 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1175,16 +1175,9 @@ end GC.safepoint() - mktemp() do tmppath, _ - open(tmppath, "w") do tmpio - redirect_stderr(tmpio) do - GC.enable_logging(true) - GC.gc() - GC.enable_logging(false) - end - end - @test occursin("GC: pause", read(open(tmppath), String)) - end + GC.enable_logging(true) + GC.gc() + GC.enable_logging(false) end @testset "fieldtypes Module" begin From cb1e40b695b246359d88c75a683a1180dc9dcced Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 7 Nov 2022 10:55:28 -0800 Subject: [PATCH 39/41] test cleanup for #44444 (#44993) (#47416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This warning was un-ironically introduced by "Fix or suppress some noisy tests". ``` ┌ Error: mktemp cleanup │ exception = │ IOError: unlink("C:\\Windows\\TEMP\\jl_A49B.tmp"): resource busy or locked (EBUSY) │ Stacktrace: │ [1] uv_error │ @ .\libuv.jl:100 [inlined] │ [2] unlink(p::String) │ @ Base.Filesystem .\file.jl:968 │ [3] rm(path::String; force::Bool, recursive::Bool) │ @ Base.Filesystem .\file.jl:283 │ [4] rm │ @ .\file.jl:274 [inlined] │ [5] mktemp(fn::Main.Test49Main_misc.var"#110#113", parent::String) │ @ Base.Filesystem .\file.jl:736 │ [6] mktemp(fn::Function) │ @ Base.Filesystem .\file.jl:730 │ [7] macro expansion │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\test\misc.jl:1065 [inlined] │ [8] macro expansion │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\stdlib\v1.9\Test\src\Test.jl:1360 [inlined] │ [9] top-level scope │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\test\misc.jl:1060 │ [10] include │ @ .\Base.jl:427 [inlined] │ [11] macro expansion │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\test\testdefs.jl:24 [inlined] │ [12] macro expansion │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\stdlib\v1.9\Test\src\Test.jl:1360 [inlined] │ [13] macro expansion │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\test\testdefs.jl:23 [inlined] │ [14] macro expansion │ @ .\timing.jl:440 [inlined] │ [15] runtests(name::String, path::String, isolate::Bool; seed::UInt128) │ @ Main C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\test\testdefs.jl:21 │ [16] invokelatest(::Any, ::Any, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, UInt128, Tuple{Symbol}, NamedTuple{(:seed,), Tuple{UInt128}}}) │ @ Base .\essentials.jl:798 │ [17] (::Distributed.var"#110#112"{Distributed.CallMsg{:call_fetch}})() │ @ Distributed C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\stdlib\v1.9\Distributed\src\process_messages.jl:285 │ [18] run_work_thunk(thunk::Distributed.var"#110#112"{Distributed.CallMsg{:call_fetch}}, print_error::Bool) │ @ Distributed C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\stdlib\v1.9\Distributed\src\process_messages.jl:70 │ [19] macro expansion │ @ C:\buildbot\worker-tabularasa\tester_win64\build\share\julia\stdlib\v1.9\Distributed\src\process_messages.jl:285 [inlined] │ [20] (::Distributed.var"#109#111"{Distributed.CallMsg{:call_fetch}, Distributed.MsgHeader, Sockets.TCPSocket})() │ @ Distributed .\task.jl:490 └ @ Base.Filesystem file.jl:738 ``` (cherry picked from commit d28a0ddd6919d37e5a4e5f1cd4857d2f83df7728) Co-authored-by: Jameson Nash --- test/misc.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/misc.jl b/test/misc.jl index 293b9d868c3b3..5d835c1dcb001 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1175,9 +1175,16 @@ end GC.safepoint() - GC.enable_logging(true) - GC.gc() - GC.enable_logging(false) + mktemp() do tmppath, _ + open(tmppath, "w") do tmpio + redirect_stderr(tmpio) do + GC.enable_logging(true) + GC.gc() + GC.enable_logging(false) + end + end + @test occursin("GC: pause", read(tmppath, String)) + end end @testset "fieldtypes Module" begin From 53cf58e7891536a49842c77fa2f0c5c7c6ccd812 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Mon, 7 Nov 2022 21:20:10 +0100 Subject: [PATCH 40/41] bump Pkg to latest v1.8 --- .../Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/md5 | 1 - .../Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/sha512 | 1 - .../Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/md5 | 1 + .../Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/md5 create mode 100644 deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/sha512 diff --git a/deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/md5 b/deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/md5 deleted file mode 100644 index 23240c5a01673..0000000000000 --- a/deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -8bd9c967dc50430afdd890b967b2d776 diff --git a/deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/sha512 b/deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/sha512 deleted file mode 100644 index a808d9ea4410b..0000000000000 --- a/deps/checksums/Pkg-0027cb18c39dfa2f0bf649fda654315e11be7bc3.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -f6ada1d6bd99b25edd4411100cb8adb2918c2e58b4e33d2623c1044009f815f2addeeab3f519cf0e60d0eabaf66f1c8d45c6bd45a86837c9f3377b808483e849 diff --git a/deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/md5 b/deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/md5 new file mode 100644 index 0000000000000..7fb351bc6dcd9 --- /dev/null +++ b/deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/md5 @@ -0,0 +1 @@ +1f97646bd4a79fd623faaa029c730e3c diff --git a/deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/sha512 b/deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/sha512 new file mode 100644 index 0000000000000..b3adde51f2d0e --- /dev/null +++ b/deps/checksums/Pkg-354530e4c8c3215d92ae15672e7a2bab1db62f76.tar.gz/sha512 @@ -0,0 +1 @@ +8eb01bd4690f87d69104284721ec48238f914e8dbcb91df495062794fa26e2c60505b68e990123b532b7c65bca134014a41da60d101ea7b9a711a52939a815f8 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index f97004e4bc79a..5614030a2a07c 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = release-1.8 -PKG_SHA1 = 0027cb18c39dfa2f0bf649fda654315e11be7bc3 +PKG_SHA1 = 354530e4c8c3215d92ae15672e7a2bab1db62f76 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 0a121d19493e171cda08f841175db9c862ead637 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 1 Nov 2022 14:31:11 -0400 Subject: [PATCH 41/41] cgmemmgr,macos: add missing shared_map_lock initialization (cherry picked from commit 3661a08750a50d508148c95bf2f3ba2c87747088) --- src/cgmemmgr.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index 37e02b0efccbb..49f708cd76e76 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -218,7 +218,12 @@ static _Atomic(size_t) map_offset{0}; // Hopefully no one will set a ulimit for this to be a problem... static constexpr size_t map_size_inc_default = 128 * 1024 * 1024; static size_t map_size = 0; -static uv_mutex_t shared_map_lock; +static struct _make_shared_map_lock { + uv_mutex_t mtx; + _make_shared_map_lock() { + uv_mutex_init(&mtx); + }; +} shared_map_lock; static size_t get_map_size_inc() { @@ -264,7 +269,7 @@ static void *alloc_shared_page(size_t size, size_t *id, bool exec) *id = off; size_t map_size_inc = get_map_size_inc(); if (__unlikely(off + size > map_size)) { - uv_mutex_lock(&shared_map_lock); + uv_mutex_lock(&shared_map_lock.mtx); size_t old_size = map_size; while (off + size > map_size) map_size += map_size_inc; @@ -275,7 +280,7 @@ static void *alloc_shared_page(size_t size, size_t *id, bool exec) abort(); } } - uv_mutex_unlock(&shared_map_lock); + uv_mutex_unlock(&shared_map_lock.mtx); } return create_shared_map(size, off); } @@ -313,7 +318,6 @@ ssize_t pwrite_addr(int fd, const void *buf, size_t nbyte, uintptr_t addr) // Use `get_self_mem_fd` which has a guard to call this only once. static int _init_self_mem() { - uv_mutex_init(&shared_map_lock); struct utsname kernel; uname(&kernel); int major, minor;