From db90581bdee5ad9ca6f847e147d4cdd252210063 Mon Sep 17 00:00:00 2001 From: cossio Date: Sun, 31 Oct 2021 11:37:43 +0100 Subject: [PATCH 1/3] spacing --- src/utils.jl | 8 ++++---- test/utils.jl | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 22757d4cf4..8d9d22a2c2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -285,7 +285,7 @@ sparse_init(rng::AbstractRNG; init_kwargs...) = (dims...; kwargs...) -> sparse_i """ identity_init([rng=GLOBAL_RNG], dims...; gain=1, shift=0) -Return an `Array` of size `dims` which yields an identity mapping when used as parameters in +Return an `Array` of size `dims` which yields an identity mapping when used as parameters in most Flux layers. Use `gain` to scale the identity by a constant. Often useful in the context of transfer learning, i.e when one wants to add more capacity to @@ -297,10 +297,10 @@ Equivalent to `Base.circshift(identity(dims...), shift)`. Some caveats: Not all layers will be identity mapping when used with this init. Exceptions include recurrent layers, `DepthwiseConv` and normalization layers. -Also note that layers must have `input_size == output_size` for identity mapping to be +Also note that layers must have `input_size == output_size` for identity mapping to be possible. When this is not the case, extra dimensions of the array are padded with zeros. -For convolutional layers, in addition to the above, the kernel sizes must also be odd and +For convolutional layers, in addition to the above, the kernel sizes must also be odd and padding must be applied so that output feature maps have the same size as input feature maps, e.g by using [`SamePad`](@ref). @@ -574,7 +574,7 @@ See also [`unstack`](@ref). # Examples ```jldoctest -julia> Flux.unbatch([1 3 5 7; +julia> Flux.unbatch([1 3 5 7; 2 4 6 8]) 4-element Vector{Vector{Int64}}: [1, 2] diff --git a/test/utils.jl b/test/utils.jl index 555684b8a4..ec33ba0bc4 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,6 +1,6 @@ using Flux using Flux: throttle, nfan, glorot_uniform, glorot_normal, - kaiming_normal, kaiming_uniform, orthogonal, + kaiming_normal, kaiming_uniform, orthogonal, sparse_init, stack, unstack, Zeros, batch, unbatch using StatsBase: var, std using Random @@ -178,10 +178,10 @@ end @testset "$layer ID mapping with kernelsize $kernelsize" for layer in (Conv, ConvTranspose, CrossCor), kernelsize in ( (1,), - (3,), - (1, 3), - (3, 5), - (3, 5, 7)) + (3,), + (1, 3), + (3, 5), + (3, 5, 7)) nch = 3 l = layer(kernelsize, nch=>nch, init=identity_init, pad=SamePad()) @@ -333,9 +333,9 @@ end @testset "Batching" begin - stacked_array=[ 8 9 3 5 - 9 6 6 9 - 9 1 7 2 + stacked_array=[ 8 9 3 5 + 9 6 6 9 + 9 1 7 2 7 4 10 6 ] unstacked_array=[[8, 9, 9, 7], [9, 6, 1, 4], [3, 6, 7, 10], [5, 9, 2, 6]] @test unbatch(stacked_array) == unstacked_array @@ -445,7 +445,7 @@ end modules = Flux.modules(Chain(SkipConnection( Conv((2,3), 4=>5; pad=6, stride=7), - +), + +), LayerNorm(8))) @test length(modules) == 5 end @@ -475,16 +475,16 @@ end @testset "early stopping" begin @testset "args & kwargs" begin es = Flux.early_stopping((x; y = 1) -> x + y, 10; min_dist=3) - + n_iter = 0 while n_iter < 99 es(-n_iter; y=-n_iter) && break n_iter += 1 end - + @test n_iter == 9 end - + @testset "distance" begin es = Flux.early_stopping(identity, 10; distance=(best_score, score) -> score - best_score) @@ -496,7 +496,7 @@ end @test n_iter == 99 end - + @testset "init_score" begin es = Flux.early_stopping(identity, 10; init_score=10) From 91d42a958dd0ca58120900de15187473d4b2d896 Mon Sep 17 00:00:00 2001 From: cossio Date: Sun, 31 Oct 2021 11:38:38 +0100 Subject: [PATCH 2/3] unsqueeze tests --- test/utils.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/utils.jl b/test/utils.jl index ec33ba0bc4..e94460d1e3 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,11 +1,20 @@ using Flux using Flux: throttle, nfan, glorot_uniform, glorot_normal, kaiming_normal, kaiming_uniform, orthogonal, - sparse_init, stack, unstack, Zeros, batch, unbatch + sparse_init, stack, unstack, Zeros, batch, unbatch, + unsqueeze using StatsBase: var, std using Random using Test +@testset "unsqueeze" begin + x = randn(2, 3, 2) + @test unsqueeze(x, 1) == reshape(x, 1, 2, 3, 2) + @test unsqueeze(x, 2) == reshape(x, 2, 1, 3, 2) + @test unsqueeze(x, 3) == reshape(x, 2, 3, 1, 2) + @test unsqueeze(x, 4) == reshape(x, 2, 3, 2, 1) +end + @testset "Throttle" begin @testset "default behaviour" begin a = [] From 78dd3f6f14bfb9be73d59edd9e4749703247a71c Mon Sep 17 00:00:00 2001 From: cossio Date: Sun, 31 Oct 2021 11:39:42 +0100 Subject: [PATCH 3/3] make unsqueeze type stable --- src/utils.jl | 5 ++++- test/utils.jl | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 8d9d22a2c2..c1888829d4 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -420,7 +420,10 @@ julia> Flux.unsqueeze(xs, 1) [1, 2] [3, 4] [5, 6] ``` """ -unsqueeze(xs::AbstractArray, dim::Integer) = reshape(xs, (size(xs)[1:dim-1]..., 1, size(xs)[dim:end]...)) +function unsqueeze(xs::AbstractArray, dim::Integer) + sz = ntuple(i -> i < dim ? size(xs, i) : i == dim ? 1 : size(xs, i - 1), ndims(xs) + 1) + return reshape(xs, sz) +end """ unsqueeze(dim) diff --git a/test/utils.jl b/test/utils.jl index e94460d1e3..9ef8da663d 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -9,10 +9,10 @@ using Test @testset "unsqueeze" begin x = randn(2, 3, 2) - @test unsqueeze(x, 1) == reshape(x, 1, 2, 3, 2) - @test unsqueeze(x, 2) == reshape(x, 2, 1, 3, 2) - @test unsqueeze(x, 3) == reshape(x, 2, 3, 1, 2) - @test unsqueeze(x, 4) == reshape(x, 2, 3, 2, 1) + @test @inferred(unsqueeze(x, 1)) == reshape(x, 1, 2, 3, 2) + @test @inferred(unsqueeze(x, 2)) == reshape(x, 2, 1, 3, 2) + @test @inferred(unsqueeze(x, 3)) == reshape(x, 2, 3, 1, 2) + @test @inferred(unsqueeze(x, 4)) == reshape(x, 2, 3, 2, 1) end @testset "Throttle" begin