Skip to content

Commit

Permalink
[Rootfs] Improve API of expand_microarchitectures (#232)
Browse files Browse the repository at this point in the history 
Add a positional argument to select the microarchitectures to expand, and a
keyword argument to select the platforms to expand.  This should provide a
flexible interface to limit the expansion and avoid building for many useless
platforms.
  • Loading branch information
@giordano
giordano authored Mar 20, 2022
1 parent 3e0815e commit 55c795d
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 31 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "BinaryBuilderBase"
uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
authors = ["Elliot Saba <staticfloat@gmail.com>"]
version = "1.7.0"
version = "1.8.0"

[deps]
CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
Expand Down
103 changes: 78 additions & 25 deletions src/Rootfs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -675,7 +675,7 @@ end
choose_shards(::AnyPlatform; kwargs...) = choose_shards(default_host_platform; kwargs...)

"""
supported_platforms(;exclude::Union{Vector{<:Platform},Function}=x->false,
supported_platforms(;exclude::Union{Vector{<:Platform},Function}=Returns(false),
experimental::Bool=false)
Return the list of supported platforms as an array of `Platform`s. These are the platforms we
Expand All @@ -690,7 +690,7 @@ or a function that returns true for exclusions i.e.
supported_platforms(exclude=Sys.islinux)
```
"""
function supported_platforms(;exclude::Union{Vector{<:Platform},Function}=x->false,
function supported_platforms(;exclude::Union{Vector{<:Platform},Function}=Returns(false),
experimental::Bool=false)
exclude_platforms!(platforms, exclude::Function) = filter(!exclude, platforms)
exclude_platforms!(platforms, exclude::Vector{<:Platform}) = filter!(!in(exclude), platforms)
Expand Down Expand Up @@ -796,11 +796,12 @@ function expand_cxxstring_abis(ps::Vector{<:AbstractPlatform}; kwargs...)
end

"""
expand_microarchitectures(p::AbstractPlatform)
expand_microarchitectures(p::AbstractPlatform, [microarchitectures::Vector{String}])
Given a `Platform`, returns a vector of `Platforms` with differing `march` attributes
as specified by the `ARCHITECTURE_FLAGS` mapping. If the given `Platform` alread has a
`march` tag specified, only that platform is returned.
`march` tag specified, only that platform is returned. If the `microarchitectures`
argument is given, limit the expansion to the given microarchitectures.
```jldoctest
julia> using BinaryBuilderBase
Expand All @@ -817,28 +818,30 @@ julia> expand_microarchitectures(Platform("armv7l", "linux"))
Linux armv7l {call_abi=eabihf, libc=glibc, march=armv7l}
Linux armv7l {call_abi=eabihf, libc=glibc, march=neonvfpv4}
julia> expand_microarchitectures(Platform("aarch64", "linux"))
4-element Vector{Platform}:
julia> expand_microarchitectures(Platform("aarch64", "linux"), ["armv8_0", "a64fx"])
2-element Vector{Platform}:
Linux aarch64 {libc=glibc, march=armv8_0}
Linux aarch64 {libc=glibc, march=armv8_4_crypto_sve}
Linux aarch64 {libc=glibc, march=armv8_2_crypto}
Linux aarch64 {libc=glibc, march=armv8_1}
Linux aarch64 {libc=glibc, march=a64fx}
julia> expand_microarchitectures(Platform("i686", "windows"))
2-element Vector{Platform}:
Windows i686 {march=pentium4}
Windows i686 {march=prescott}
```
"""
function expand_microarchitectures(platform::AbstractPlatform)
# If this already has a `march`, or it's an `AnyPlatform`, just return it.
if isa(platform, AnyPlatform) || march(platform) !== nothing
function expand_microarchitectures(platform::AbstractPlatform,
microarchitectures::Vector{String}=get_all_march_names(arch(platform)))
all_marchs = get_all_march_names(arch(platform))

# If this already has a `march`, or it's an `AnyPlatform`, or the microarchitectures we
# want to expand aren't relative to this platform, just return it.
if isa(platform, AnyPlatform) || march(platform) !== nothing ||
!any(in(all_marchs), microarchitectures)
return [platform]
end

# Otherwise, return a bunch of Platform objects with appropriately-set `march` tags, but
# first filter out some meaningless combinations of microarchitectures.
all_marchs = filter(get_all_march_names(arch(platform))) do march
# First, filter out some meaningless combinations of microarchitectures.
marchs = filter(all_marchs) do march
if (!Sys.isapple(platform) && march == "apple_m1") ||
(Sys.isapple(platform) && arch(platform) == "aarch64" && march ∉ ("armv8_0", "apple_m1"))
# `apple_m1` makes sense only on macOS, and the only aarch64 microarchitectures
Expand All @@ -851,39 +854,89 @@ function expand_microarchitectures(platform::AbstractPlatform)
return true
end

return map(all_marchs) do march
p = deepcopy(platform)
p["march"] = march
return p
# But if none of the remaining microarchitectures are among those we want to expand,
# return the given platform as is.
if !any(in(microarchitectures), marchs)
return [platform]
end

# Otherwise, return a bunch of Platform objects with appropriately-set `march` tags
return [(p = deepcopy(platform); p["march"] = march; p) for march in marchs if march in microarchitectures]
end

"""
expand_microarchitectures(ps::Vector{<:Platform})
expand_microarchitectures(ps::Vector{<:Platform}, [microarchitectures::Vector{String}];
filter=Returns(true))
Expand all platforms in the vector `ps` with the supported microarchitectures.
Expand all platforms in the given vector with the supported microarchitectures.
If the `microarchitectures` argument is given, limit the expansion to the given
platforms. This is useful if you do not want to expand to all available
microarchitectures.
The expansion is applied only to the platforms matching the `filter` predicate, by
default all platforms. This is useful if you want to limit the expansion to some
platforms, without having to explicitly list its microarchitectures in the second
argument.
```jldoctest
julia> using BinaryBuilderBase
julia> expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms()))
13-element Vector{Platform}:
14-element Vector{Platform}:
Linux i686 {libc=glibc, march=pentium4}
Linux i686 {libc=glibc, march=prescott}
Linux x86_64 {libc=glibc, march=x86_64}
Linux x86_64 {libc=glibc, march=avx}
Linux x86_64 {libc=glibc, march=avx2}
Linux x86_64 {libc=glibc, march=avx512}
Linux aarch64 {libc=glibc, march=armv8_0}
Linux aarch64 {libc=glibc, march=armv8_4_crypto_sve}
Linux aarch64 {libc=glibc, march=armv8_2_crypto}
Linux aarch64 {libc=glibc, march=armv8_1}
Linux aarch64 {libc=glibc, march=armv8_2_crypto}
Linux aarch64 {libc=glibc, march=a64fx}
Linux armv6l {call_abi=eabihf, libc=glibc, march=arm1176jzfs}
Linux armv7l {call_abi=eabihf, libc=glibc, march=armv7l}
Linux armv7l {call_abi=eabihf, libc=glibc, march=neonvfpv4}
Linux powerpc64le {libc=glibc, march=power8}
julia> expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms()), ["x86_64", "avx2"])
7-element Vector{Platform}:
Linux i686 {libc=glibc}
Linux x86_64 {libc=glibc, march=x86_64}
Linux x86_64 {libc=glibc, march=avx2}
Linux aarch64 {libc=glibc}
Linux armv6l {call_abi=eabihf, libc=glibc}
Linux armv7l {call_abi=eabihf, libc=glibc}
Linux powerpc64le {libc=glibc}
julia> expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms()); filter=p->arch(p)=="x86_64")
9-element Vector{Platform}:
Linux i686 {libc=glibc}
Linux x86_64 {libc=glibc, march=x86_64}
Linux x86_64 {libc=glibc, march=avx}
Linux x86_64 {libc=glibc, march=avx2}
Linux x86_64 {libc=glibc, march=avx512}
Linux aarch64 {libc=glibc}
Linux armv6l {call_abi=eabihf, libc=glibc}
Linux armv7l {call_abi=eabihf, libc=glibc}
Linux powerpc64le {libc=glibc}
```
"""
expand_microarchitectures(ps::Vector{<:AbstractPlatform}) = collect(Iterators.flatten(expand_microarchitectures.(ps)))
function expand_microarchitectures(ps::Vector{<:AbstractPlatform},
microarchitectures::Vector{String}=collect(Iterators.flatten(get_all_march_names.(unique!(arch.(ps)))));
filter=Returns(true),
)
out = map(ps) do p
return if filter(p)
# If the platform satisfies the predicate, expand its microarchitectures
expand_microarchitectures(p, microarchitectures)
else
# otherwise return it as-is.
[p]
end
end
return collect(Iterators.flatten(out))
end

"""
preferred_libgfortran_version(platform::AbstractPlatform, shard::CompilerShard;
Expand Down
19 changes: 14 additions & 5 deletions test/rootfs.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,14 @@ using BinaryBuilderBase
Platform("x86_64", "linux"; libc="glibc", march="avx512", cuda="10.1"),
Platform("x86_64", "linux"; libc="glibc", march="x86_64", cuda="10.1"),
]
@test sort(expand_microarchitectures(Platform("x86_64", "linux"), ["x86_64", "avx512"]), by=triplet) == [
Platform("x86_64", "linux"; libc="glibc", march="avx512"),
Platform("x86_64", "linux"; libc="glibc", march="x86_64"),
]
@test expand_microarchitectures(Platform("i686", "linux"), ["x86_64", "avx512"]) == [Platform("i686", "linux")]
@test expand_microarchitectures(Platform("aarch64", "linux"; libc="musl"), ["a64fx", "apple_m1"]) == [Platform("aarch64", "linux"; libc="musl")]
@test expand_microarchitectures(Platform("aarch64", "macos"), ["a64fx"]) == [Platform("aarch64", "macos")]

@test sort(expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "glibc", supported_platforms())), by=triplet) == [
Platform("aarch64", "linux"; libc="glibc", march="a64fx"),
Platform("aarch64", "linux"; libc="glibc", march="armv8_0"),
Expand All @@ -73,17 +81,18 @@ using BinaryBuilderBase
Platform("x86_64", "linux"; libc="glibc", march="avx512"),
Platform("x86_64", "linux"; libc="glibc", march="x86_64"),
]
@test sort(expand_microarchitectures(filter!(p -> Sys.islinux(p) && arch(p) == "aarch64" && libc(p) == "musl", supported_platforms())), by=triplet) == [
@test sort(expand_microarchitectures(filter!(p -> Sys.islinux(p) && libc(p) == "musl", supported_platforms()); filter=p->arch(p) == "aarch64"), by=triplet) == [
Platform("aarch64", "linux"; libc="musl", march="armv8_0"),
Platform("aarch64", "linux"; libc="musl", march="armv8_1"),
Platform("aarch64", "linux"; libc="musl", march="armv8_2_crypto"),
Platform("armv6l", "linux"; call_abi="eabihf", libc="musl"),
Platform("armv7l", "linux"; call_abi="eabihf", libc="musl"),
Platform("i686", "linux"; libc="musl"),
Platform("x86_64", "linux"; libc="musl"),
]
@test sort(expand_microarchitectures(filter!(p -> Sys.isapple(p), supported_platforms())), by=triplet) == [
@test sort(expand_microarchitectures(filter!(p -> Sys.isapple(p), supported_platforms()), ["a64fx", "apple_m1", "x86_64", "avx2"]), by=triplet) == [
Platform("aarch64", "macos"; march="apple_m1"),
Platform("aarch64", "macos"; march="armv8_0"),
Platform("x86_64", "macos"; march="avx"),
Platform("x86_64", "macos"; march="avx2"),
Platform("x86_64", "macos"; march="avx512"),
Platform("x86_64", "macos"; march="x86_64"),
]
@test expand_microarchitectures([Platform("x86_64", "windows"; march="avx", cuda="10.1")]) ==
Expand Down

2 comments on commit 55c795d

@giordano
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/56924

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v1.8.0 -m "<description of version>" 55c795dbca6c74b076636bdcfdccbfeb34063eb7
git push origin v1.8.0

Please sign in to comment.