From 79c4c8cd734651a00710f959015b24762bd46073 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 18 Nov 2014 12:50:36 -0600 Subject: [PATCH 1/6] IteratorsMD: remove ambiguity resolution for SharedArrays --- base/multidimensional.jl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 1ce679dc0e74f..d3c71e391911c 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -24,7 +24,7 @@ end let implemented = IntSet() global gen_cartesian -function gen_cartesian(N::Int, with_shared=Base.is_unix(OS_NAME)) +function gen_cartesian(N::Int) # Create the types indextype = symbol("CartesianIndex_$N") itertype = symbol("IndexIterator_$N") @@ -43,23 +43,19 @@ function gen_cartesian(N::Int, with_shared=Base.is_unix(OS_NAME)) next(R::StepRange, I::CartesianIndex_1) = R[I.I_1], CartesianIndex_1(I.I_1+1) next{T}(R::UnitRange{T}, I::CartesianIndex_1) = R[I.I_1], CartesianIndex_1(I.I_1+1) end - exshared = !with_shared ? nothing : quote - getindex{T}(S::SharedArray{T,$N}, I::$indextype) = S.s[I] - setindex!{T}(S::SharedArray{T,$N}, v, I::$indextype) = S.s[I] = v - end totalex = quote - # type definition + # type definition of state $extype # extra constructor from tuple $indextype(index::NTuple{$N,Int}) = $indextype($(exindices...)) + # type definition of iterator immutable $itertype <: IndexIterator{$N} dims::$indextype end $itertype(dims::NTuple{$N,Int})=$itertype($indextype(dims)) # getindex and setindex! - $exshared getindex{T}(A::AbstractArray{T,$N}, index::$indextype) = @nref $N A d->getfield(index,d) setindex!{T}(A::AbstractArray{T,$N}, v, index::$indextype) = (@nref $N A d->getfield(index,d)) = v From 8090e2af849c8baa97eee42e151dbe4e22231d40 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 18 Nov 2014 13:22:06 -0600 Subject: [PATCH 2/6] IteratorsMD: handle 0-dimensional arrays properly --- base/multidimensional.jl | 26 ++++++++++++++++---------- test/arrayops.jl | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index d3c71e391911c..85f5778878452 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -29,14 +29,18 @@ function gen_cartesian(N::Int) indextype = symbol("CartesianIndex_$N") itertype = symbol("IndexIterator_$N") if !in(N,implemented) - fieldnames = [symbol("I_$i") for i = 1:N] - fields = [Expr(:(::), fieldnames[i], :Int) for i = 1:N] + M = max(N,1) # 0-dimensional arrays require special handling + fieldnames = [symbol("I_$i") for i = 1:M] + fields = [Expr(:(::), fieldnames[i], :Int) for i = 1:M] extype = Expr(:type, false, Expr(:(<:), indextype, Expr(:curly, :CartesianIndex, N)), Expr(:block, fields...)) exindices = Expr[:(index[$i]) for i = 1:N] + index_tuple_constr = N > 0 ? + (:($indextype(index::NTuple{$N,Int}) = $indextype($(exindices...)))) : + (:($indextype(index::NTuple{0,Int}) = $indextype(1))) - onesN = ones(Int, N) - infsN = fill(typemax(Int), N) - anyzero = Expr(:(||), [:(iter.dims.$(fieldnames[i]) == 0) for i = 1:N]...) + onesN = ones(Int, M) + infsN = fill(typemax(Int), M) + anyzero = Expr(:(||), [:(iter.dims.$(fieldnames[i]) == 0) for i = 1:M]...) # Some necessary ambiguity resolution exrange = N != 1 ? nothing : quote @@ -46,13 +50,14 @@ function gen_cartesian(N::Int) totalex = quote # type definition of state $extype - # extra constructor from tuple - $indextype(index::NTuple{$N,Int}) = $indextype($(exindices...)) + # constructor from tuple + $index_tuple_constr # type definition of iterator immutable $itertype <: IndexIterator{$N} dims::$indextype end + # constructor from tuple $itertype(dims::NTuple{$N,Int})=$itertype($indextype(dims)) # getindex and setindex! @@ -63,11 +68,11 @@ function gen_cartesian(N::Int) $exrange @inline function next{T}(A::AbstractArray{T,$N}, state::$indextype) @inbounds v = A[state] - newstate = @nif $N d->(getfield(state,d) < size(A, d)) d->(@ncall($N, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) + newstate = @nif $M d->(getfield(state,d) < size(A, d)) d->(@ncall($N, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) v, newstate end @inline function next(iter::$itertype, state::$indextype) - newstate = @nif $N d->(getfield(state,d) < getfield(iter.dims,d)) d->(@ncall($N, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) + newstate = @nif $M d->(getfield(state,d) < getfield(iter.dims,d)) d->(@ncall($M, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) state, newstate end @@ -86,7 +91,7 @@ eachindex(A::AbstractArray) = IndexIterator(size(A)) # start iteration stagedfunction _start{T,N}(A::AbstractArray{T,N},::LinearSlow) - args = fill(:s, N) + args = fill(:s, max(N,1)) indextype, _ = gen_cartesian(N) quote s = ifelse(isempty(A), typemax(Int), 1) @@ -100,6 +105,7 @@ done(R::UnitRange, I::CartesianIndex{1}) = getfield(I, 1) > length(R) done(R::FloatRange, I::CartesianIndex{1}) = getfield(I, 1) > length(R) done{T,N}(A::AbstractArray{T,N}, I::CartesianIndex{N}) = getfield(I, N) > size(A, N) +done(iter::IndexIterator{0}, I::CartesianIndex{0}) = getfield(I, 1) > getfield(iter.dims, 1) done{N}(iter::IndexIterator{N}, I::CartesianIndex{N}) = getfield(I, N) > getfield(iter.dims, N) end # IteratorsMD diff --git a/test/arrayops.jl b/test/arrayops.jl index 00f6b5e673809..edd55872e971e 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -927,6 +927,19 @@ b718cbc = 5 @test_throws InexactError b718cbc[1.1] # Multidimensional iterators +for a in ([1:5], reshape([2])) + counter = 0 + for I in eachindex(a) + counter += 1 + end + @test counter == length(a) + counter = 0 + for aa in a + counter += 1 + end + @test counter == length(a) +end + function mdsum(A) s = 0.0 for a in A @@ -970,9 +983,14 @@ for i = 2:10 insert!(shp, 2, 1) end +a = reshape([2]) +@test mdsum(a) == 2 +@test mdsum2(a) == 2 + a = ones(0,5) b = sub(a, :, :) @test mdsum(b) == 0 a = ones(5,0) b = sub(a, :, :) @test mdsum(b) == 0 + From 374e0938c4ed3f04323ed9b2aa8d8b5ad1892865 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 19 Nov 2014 08:56:41 -0600 Subject: [PATCH 3/6] IteratorsMD: allow AbstractArray methods to be overriden Any AbstractArray type that wants to override an indexing methods would have formerly run into trouble: next{T,N}(A::SomeArrayType{T,N}, state::CartesianIndex{N}) would be more specific in the first argument, but (because CartesianIndex{N} is less specific than CartesianIndex_$N) would be less specific in the second argument. So there was no way to avoid a method ambiguity warning. --- base/multidimensional.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 85f5778878452..a516875910b95 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -44,8 +44,8 @@ function gen_cartesian(N::Int) # Some necessary ambiguity resolution exrange = N != 1 ? nothing : quote - next(R::StepRange, I::CartesianIndex_1) = R[I.I_1], CartesianIndex_1(I.I_1+1) - next{T}(R::UnitRange{T}, I::CartesianIndex_1) = R[I.I_1], CartesianIndex_1(I.I_1+1) + next(R::StepRange, I::CartesianIndex{1}) = R[I.I_1], CartesianIndex_1(I.I_1+1) + next{T}(R::UnitRange{T}, I::CartesianIndex{1}) = R[I.I_1], CartesianIndex_1(I.I_1+1) end totalex = quote # type definition of state @@ -61,17 +61,17 @@ function gen_cartesian(N::Int) $itertype(dims::NTuple{$N,Int})=$itertype($indextype(dims)) # getindex and setindex! - getindex{T}(A::AbstractArray{T,$N}, index::$indextype) = @nref $N A d->getfield(index,d) - setindex!{T}(A::AbstractArray{T,$N}, v, index::$indextype) = (@nref $N A d->getfield(index,d)) = v + getindex{T}(A::AbstractArray{T,$N}, index::CartesianIndex{$N}) = @nref $N A d->getfield(index,d) + setindex!{T}(A::AbstractArray{T,$N}, v, index::CartesianIndex{$N}) = (@nref $N A d->getfield(index,d)) = v # next iteration $exrange - @inline function next{T}(A::AbstractArray{T,$N}, state::$indextype) + @inline function next{T}(A::AbstractArray{T,$N}, state::CartesianIndex{$N}) @inbounds v = A[state] newstate = @nif $M d->(getfield(state,d) < size(A, d)) d->(@ncall($N, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) v, newstate end - @inline function next(iter::$itertype, state::$indextype) + @inline function next(iter::$itertype, state::CartesianIndex{$N}) newstate = @nif $M d->(getfield(state,d) < getfield(iter.dims,d)) d->(@ncall($M, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) state, newstate end From 9b4f2123d22d662c16cba46d46c091f19aba8bf3 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 19 Nov 2014 08:58:55 -0600 Subject: [PATCH 4/6] IteratorsMD: document eachindex and add NEWS.md item --- NEWS.md | 3 +++ doc/stdlib/base.rst | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/NEWS.md b/NEWS.md index 0560ee6d447c1..5b32e3d4a3e43 100644 --- a/NEWS.md +++ b/NEWS.md @@ -24,6 +24,9 @@ New language features and macros in packages and user code ([#8791]). Type `?@doc` at the repl to see the current syntax and more information. + * New multidimensional iterators and index types for efficient + iteration over general AbstractArrays + Language changes ---------------- diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 721caf776da8d..7954a1afd4f72 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -4066,6 +4066,32 @@ Basic functions Returns the number of elements in A +.. function:: eachindex(A) + + Creates an iterable object for visiting each multi-dimensional index of the AbstractArray ``A``. Example for a 2-d array:: + + julia> A = rand(2,3) + 2x3 Array{Float64,2}: + 0.960084 0.629326 0.625155 + 0.432588 0.955903 0.991614 + + julia> for iter in eachindex(A) + @show iter.I_1, iter.I_2 + @show A[iter] + end + (iter.I_1,iter.I_2) = (1,1) + A[iter] = 0.9600836263003063 + (iter.I_1,iter.I_2) = (2,1) + A[iter] = 0.4325878255452178 + (iter.I_1,iter.I_2) = (1,2) + A[iter] = 0.6293256402775211 + (iter.I_1,iter.I_2) = (2,2) + A[iter] = 0.9559027084099654 + (iter.I_1,iter.I_2) = (1,3) + A[iter] = 0.6251548453735303 + (iter.I_1,iter.I_2) = (2,3) + A[iter] = 0.9916142534546522 + .. function:: countnz(A) Counts the number of nonzero values in array A (dense or sparse). Note that this is not a constant-time operation. For sparse matrices, one should usually use ``nnz``, which returns the number of stored values. From 5557a99418bfca50a051c51b56448a2e47958c68 Mon Sep 17 00:00:00 2001 From: Jutho Haegeman Date: Wed, 19 Nov 2014 15:23:46 -0600 Subject: [PATCH 5/6] IteratorsMD: avoid ambiguity warnings by using more stagedfunctions This also implements a way of handling 0-dimensional arrays that doesn't require adding a field to a CartesianIndex_0. --- base/multidimensional.jl | 114 ++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 49 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index a516875910b95..e7011faf8f241 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -14,7 +14,7 @@ abstract CartesianIndex{N} # the state for all multidimensional iterators abstract IndexIterator{N} # Iterator that visits the index associated with each element stagedfunction Base.call{N}(::Type{CartesianIndex},index::NTuple{N,Int}) - indextype,itertype=gen_cartesian(N) + indextype, itertype = gen_cartesian(N) return :($indextype(index)) end stagedfunction Base.call{N}(::Type{IndexIterator},index::NTuple{N,Int}) @@ -29,29 +29,15 @@ function gen_cartesian(N::Int) indextype = symbol("CartesianIndex_$N") itertype = symbol("IndexIterator_$N") if !in(N,implemented) - M = max(N,1) # 0-dimensional arrays require special handling - fieldnames = [symbol("I_$i") for i = 1:M] - fields = [Expr(:(::), fieldnames[i], :Int) for i = 1:M] + fieldnames = [symbol("I_$i") for i = 1:N] + fields = [Expr(:(::), fieldnames[i], :Int) for i = 1:N] extype = Expr(:type, false, Expr(:(<:), indextype, Expr(:curly, :CartesianIndex, N)), Expr(:block, fields...)) exindices = Expr[:(index[$i]) for i = 1:N] - index_tuple_constr = N > 0 ? - (:($indextype(index::NTuple{$N,Int}) = $indextype($(exindices...)))) : - (:($indextype(index::NTuple{0,Int}) = $indextype(1))) - - onesN = ones(Int, M) - infsN = fill(typemax(Int), M) - anyzero = Expr(:(||), [:(iter.dims.$(fieldnames[i]) == 0) for i = 1:M]...) - - # Some necessary ambiguity resolution - exrange = N != 1 ? nothing : quote - next(R::StepRange, I::CartesianIndex{1}) = R[I.I_1], CartesianIndex_1(I.I_1+1) - next{T}(R::UnitRange{T}, I::CartesianIndex{1}) = R[I.I_1], CartesianIndex_1(I.I_1+1) - end totalex = quote # type definition of state $extype # constructor from tuple - $index_tuple_constr + $indextype(index::NTuple{$N,Int}) = $indextype($(exindices...)) # type definition of iterator immutable $itertype <: IndexIterator{$N} @@ -59,25 +45,6 @@ function gen_cartesian(N::Int) end # constructor from tuple $itertype(dims::NTuple{$N,Int})=$itertype($indextype(dims)) - - # getindex and setindex! - getindex{T}(A::AbstractArray{T,$N}, index::CartesianIndex{$N}) = @nref $N A d->getfield(index,d) - setindex!{T}(A::AbstractArray{T,$N}, v, index::CartesianIndex{$N}) = (@nref $N A d->getfield(index,d)) = v - - # next iteration - $exrange - @inline function next{T}(A::AbstractArray{T,$N}, state::CartesianIndex{$N}) - @inbounds v = A[state] - newstate = @nif $M d->(getfield(state,d) < size(A, d)) d->(@ncall($N, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) - v, newstate - end - @inline function next(iter::$itertype, state::CartesianIndex{$N}) - newstate = @nif $M d->(getfield(state,d) < getfield(iter.dims,d)) d->(@ncall($M, $indextype, k->(k>d ? getfield(state,k) : k==d ? getfield(state,k)+1 : 1))) - state, newstate - end - - # start - start(iter::$itertype) = $anyzero ? $indextype($(infsN...)) : $indextype($(onesN...)) end eval(totalex) push!(implemented,N) @@ -86,27 +53,76 @@ function gen_cartesian(N::Int) end end -# Iteration +# indexing +stagedfunction getindex{N}(A::AbstractArray, index::CartesianIndex{N}) + :(@nref $N A d->getfield(index,d)) +end +stagedfunction setindex!{N}(A::AbstractArray, v, index::CartesianIndex{N}) + :((@nref $N A d->getfield(index,d)) = v) +end + +# Prevent an ambiguity warning +gen_cartesian(1) # to make sure the next two lines are valid +next(R::StepRange, state::(Bool, CartesianIndex{1})) = R[state[2].I_1], (state[2].I_1==length(R), CartesianIndex_1(state[2].I_1+1)) +next{T}(R::UnitRange{T}, state::(Bool, CartesianIndex{1})) = R[state[2].I_1], (state[2].I_1==length(R), CartesianIndex_1(state[2].I_1+1)) + +# iteration eachindex(A::AbstractArray) = IndexIterator(size(A)) -# start iteration +stagedfunction start{N}(iter::IndexIterator{N}) + indextype, _ = gen_cartesian(N) + args = fill(:s, N) + fieldnames = [symbol("I_$i") for i = 1:N] + anyzero = Expr(:(||), [:(iter.dims.$(fieldnames[i]) == 0) for i = 1:N]...) + quote + z = $anyzero + s = ifelse(z, typemax(Int), 1) + return z, $indextype($(args...)) + end +end + stagedfunction _start{T,N}(A::AbstractArray{T,N},::LinearSlow) - args = fill(:s, max(N,1)) indextype, _ = gen_cartesian(N) + args = fill(:s, N) + quote + z = isempty(A) + s = ifelse(z, typemax(Int), 1) + return z, $indextype($(args...)) + end +end + +stagedfunction next{T,N}(A::AbstractArray{T,N}, state::(Bool, CartesianIndex{N})) + indextype, _ = gen_cartesian(N) + finishedex = (N==0 ? true : :(getfield(newindex, $N) > size(A, $N))) + meta = Expr(:meta, :inline) + quote + $meta + index=state[2] + @inbounds v = A[index] + newindex=@nif $N d->(getfield(index,d) < size(A, d)) d->@ncall($N, $indextype, k->(k>d ? getfield(index,k) : k==d ? getfield(index,k)+1 : 1)) + finished=$finishedex + v, (finished,newindex) + end +end +stagedfunction next{N}(iter::IndexIterator{N}, state::(Bool, CartesianIndex{N})) + indextype, _ = gen_cartesian(N) + finishedex = (N==0 ? true : :(getfield(newindex, $N) > getfield(iter.dims, $N))) + meta = Expr(:meta, :inline) quote - s = ifelse(isempty(A), typemax(Int), 1) - $indextype($(args...)) + $meta + index=state[2] + newindex=@nif $N d->(getfield(index,d) < getfield(iter.dims, d)) d->@ncall($N, $indextype, k->(k>d ? getfield(index,k) : k==d ? getfield(index,k)+1 : 1)) + finished=$finishedex + index, (finished,newindex) end end -# Ambiguity resolution -done(R::StepRange, I::CartesianIndex{1}) = getfield(I, 1) > length(R) -done(R::UnitRange, I::CartesianIndex{1}) = getfield(I, 1) > length(R) -done(R::FloatRange, I::CartesianIndex{1}) = getfield(I, 1) > length(R) +done(R::StepRange, state::(Bool, CartesianIndex{1})) = state[1] +done(R::UnitRange, state::(Bool, CartesianIndex{1})) = state[1] +done(R::FloatRange, state::(Bool, CartesianIndex{1})) = state[1] -done{T,N}(A::AbstractArray{T,N}, I::CartesianIndex{N}) = getfield(I, N) > size(A, N) -done(iter::IndexIterator{0}, I::CartesianIndex{0}) = getfield(I, 1) > getfield(iter.dims, 1) -done{N}(iter::IndexIterator{N}, I::CartesianIndex{N}) = getfield(I, N) > getfield(iter.dims, N) +done{T,N}(A::AbstractArray{T,N}, state::(Bool, CartesianIndex{N})) = state[1] +done{N}(iter::IndexIterator{N}, state::(Bool, CartesianIndex{N})) = state[1] end # IteratorsMD From 24ea5e1b113bcf3cfe657d54066daf8d0d7e6c29 Mon Sep 17 00:00:00 2001 From: Jutho Date: Thu, 20 Nov 2014 09:48:56 +0100 Subject: [PATCH 6/6] IteratorsMD: small final fixes --- base/multidimensional.jl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index e7011faf8f241..3d7b485b72320 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -18,7 +18,7 @@ stagedfunction Base.call{N}(::Type{CartesianIndex},index::NTuple{N,Int}) return :($indextype(index)) end stagedfunction Base.call{N}(::Type{IndexIterator},index::NTuple{N,Int}) - indextype,itertype=gen_cartesian(N) + indextype, itertype = gen_cartesian(N) return :($itertype(index)) end @@ -71,22 +71,20 @@ eachindex(A::AbstractArray) = IndexIterator(size(A)) stagedfunction start{N}(iter::IndexIterator{N}) indextype, _ = gen_cartesian(N) - args = fill(:s, N) + args = fill(1, N) fieldnames = [symbol("I_$i") for i = 1:N] anyzero = Expr(:(||), [:(iter.dims.$(fieldnames[i]) == 0) for i = 1:N]...) quote z = $anyzero - s = ifelse(z, typemax(Int), 1) return z, $indextype($(args...)) end end -stagedfunction _start{T,N}(A::AbstractArray{T,N},::LinearSlow) +stagedfunction _start{T,N}(A::AbstractArray{T,N}, ::LinearSlow) indextype, _ = gen_cartesian(N) - args = fill(:s, N) + args = fill(1, N) quote z = isempty(A) - s = ifelse(z, typemax(Int), 1) return z, $indextype($(args...)) end end