diff --git a/Makefile b/Makefile index 8b0cc23a36227..72c832525c1c3 100644 --- a/Makefile +++ b/Makefile @@ -198,7 +198,6 @@ CORE_SRCS := $(addprefix $(JULIAHOME)/, \ base/inference.jl \ base/int.jl \ base/intset.jl \ - base/iterator.jl \ base/nofloat_hashing.jl \ base/number.jl \ base/operators.jl \ diff --git a/base/REPLCompletions.jl b/base/REPLCompletions.jl index f07b8d454cce4..437d2e0924a40 100644 --- a/base/REPLCompletions.jl +++ b/base/REPLCompletions.jl @@ -376,10 +376,10 @@ function bslash_completions(string, pos) # return possible matches; these cannot be mixed with regular # Julian completions as only latex / emoji symbols contain the leading \ if startswith(s, "\\:") # emoji - emoji_names = filter(k -> startswith(k, s), keys(emoji_symbols)) + emoji_names = IterTools.filter(k -> startswith(k, s), keys(emoji_symbols)) return (true, (sort!(collect(emoji_names)), slashpos:pos, true)) else # latex - latex_names = filter(k -> startswith(k, s), keys(latex_symbols)) + latex_names = IterTools.filter(k -> startswith(k, s), keys(latex_symbols)) return (true, (sort!(collect(latex_names)), slashpos:pos, true)) end end diff --git a/base/asyncmap.jl b/base/asyncmap.jl index 9490f713267a5..fc18f36f6c30d 100644 --- a/base/asyncmap.jl +++ b/base/asyncmap.jl @@ -1,5 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +using Base.IterTools.Enumerate """ AsyncCollector(f, results, c...; ntasks=0) -> iterator diff --git a/base/broadcast.jl b/base/broadcast.jl index 917a9be9f3a5a..6260f84f126f8 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -226,7 +226,7 @@ end else R = typejoin(eltype(B), S) new = similar(B, R) - for II in take(iter, count) + for II in IterTools.take(iter, count) new[II] = B[II] end new[I] = V diff --git a/base/client.jl b/base/client.jl index 101340145b539..ae89777adb514 100644 --- a/base/client.jl +++ b/base/client.jl @@ -23,7 +23,7 @@ end # Create a docstring with an automatically generated list # of colors. const possible_formatting_symbols = [:normal, :bold] -available_text_colors = collect(filter(x -> !isa(x, Integer), keys(text_colors))) +available_text_colors = collect(IterTools.filter(x -> !isa(x, Integer), keys(text_colors))) available_text_colors = cat(1, sort(intersect(available_text_colors, possible_formatting_symbols), rev=true), sort(setdiff( available_text_colors, possible_formatting_symbols))) diff --git a/base/coreimg.jl b/base/coreimg.jl index 288336da3be06..c6941ff4a5529 100644 --- a/base/coreimg.jl +++ b/base/coreimg.jl @@ -61,7 +61,6 @@ include("reduce.jl") ## core structures include("intset.jl") include("dict.jl") -include("iterator.jl") # core docsystem include("docs/core.jl") diff --git a/base/dates/Dates.jl b/base/dates/Dates.jl index ff2165de586d0..90d534d05447b 100644 --- a/base/dates/Dates.jl +++ b/base/dates/Dates.jl @@ -4,6 +4,8 @@ module Dates importall ..Base.Operators +using Base.IterTools + include("types.jl") include("periods.jl") include("accessors.jl") diff --git a/base/deprecated.jl b/base/deprecated.jl index a4756fa2c1ef1..c06719044cd85 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1019,4 +1019,14 @@ eval(Multimedia, :(macro textmime(mime) end end)) +@deprecate_binding Filter IterTools.Filter +@deprecate_binding Zip IterTools.Zip +@deprecate filter(flt, itr) IterTools.filter(flt, itr) +@deprecate_binding rest IterTools.rest +@deprecate_binding countfrom IterTools.countfrom +@deprecate_binding take IterTools.take +@deprecate_binding drop IterTools.drop +@deprecate_binding cycle IterTools.cycle +@deprecate_binding repeated IterTools.repeated + # End deprecations scheduled for 0.6 diff --git a/base/essentials.jl b/base/essentials.jl index 44aef5b3d85be..a5279bffbe1fd 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -63,6 +63,12 @@ function tuple_type_tail(T::DataType) return Tuple{argtail(T.parameters...)...} end +tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{} +function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) + @_pure_meta + Tuple{S, T.parameters...} +end + isvarargtype(t::ANY) = isa(t, DataType) && is((t::DataType).name, Vararg.name) isvatuple(t::DataType) = (n = length(t.parameters); n > 0 && isvarargtype(t.parameters[n])) unwrapva(t::ANY) = isvarargtype(t) ? t.parameters[1] : t @@ -231,3 +237,5 @@ function vector_any(xs::ANY...) end a end + +isempty(itr) = done(itr, start(itr)) diff --git a/base/exports.jl b/base/exports.jl index 87ca75cf646cb..5a27d381c99ec 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -23,6 +23,7 @@ export Docs, Markdown, Threads, + IterTools, # Types AbstractChannel, @@ -60,7 +61,6 @@ export Enumerate, Factorization, FileMonitor, - Filter, FloatRange, Future, Hermitian, @@ -123,7 +123,6 @@ export VersionNumber, WeakKeyDict, WorkerConfig, - Zip, # Ccall types Cchar, @@ -958,16 +957,11 @@ export # iteration done, - enumerate, next, start, + + enumerate, # re-exported from IterTools zip, - rest, - countfrom, - take, - drop, - cycle, - repeated, # object identity and equality copy, diff --git a/base/generator.jl b/base/generator.jl index fce99b73e31b7..9c1b8e0a71ab3 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -55,11 +55,6 @@ result, and algorithms that resize their result incrementally. iteratorsize(x) = iteratorsize(typeof(x)) iteratorsize(::Type) = HasLength() # HasLength is the default -and_iteratorsize{T}(isz::T, ::T) = isz -and_iteratorsize(::HasLength, ::HasShape) = HasLength() -and_iteratorsize(::HasShape, ::HasLength) = HasLength() -and_iteratorsize(a, b) = SizeUnknown() - abstract IteratorEltype immutable EltypeUnknown <: IteratorEltype end immutable HasEltype <: IteratorEltype end @@ -81,9 +76,6 @@ values. iteratoreltype(x) = iteratoreltype(typeof(x)) iteratoreltype(::Type) = HasEltype() # HasEltype is the default -and_iteratoreltype{T}(iel::T, ::T) = iel -and_iteratoreltype(a, b) = EltypeUnknown() - iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape() iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I) length(g::Generator) = length(g.iter) diff --git a/base/inference.jl b/base/inference.jl index d092d3bf85391..1e980ef3fb885 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1017,7 +1017,8 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable end function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv::InferenceState) - for a in drop(argtypes,1) + for i = 2:length(argtypes) + a = argtypes[i] if !(isa(a,Const) || isconstType(a,false)) return false end @@ -1053,7 +1054,7 @@ function pure_eval_call(f::ANY, argtypes::ANY, atype::ANY, vtypes::VarTable, sv: return false end - args = Any[ isa(a,Const) ? a.val : a.parameters[1] for a in drop(argtypes,1) ] + args = Any[ (a=argtypes[i]; isa(a,Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ] try return abstract_eval_constant(f(args...)) catch diff --git a/base/iterator.jl b/base/itertools.jl similarity index 95% rename from base/iterator.jl rename to base/itertools.jl index bfcf96eeaaa63..71d1ca396efdc 100644 --- a/base/iterator.jl +++ b/base/itertools.jl @@ -1,6 +1,14 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -isempty(itr) = done(itr, start(itr)) +module IterTools + +global Filter + +import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims + +using Base: tuple_type_cons, SizeUnknown, HasLength, HasShape, IsInfinite, EltypeUnknown, HasEltype, OneTo + +export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, flatten, partition _min_length(a, b, ::IsInfinite, ::IsInfinite) = min(length(a),length(b)) # inherit behaviour, error _min_length(a, b, A, ::IsInfinite) = length(a) @@ -12,6 +20,14 @@ _diff_length(a, b, ::IsInfinite, ::IsInfinite) = 0 _diff_length(a, b, ::IsInfinite, B) = length(a) # inherit behaviour, error _diff_length(a, b, A, B) = max(length(a)-length(b), 0) +and_iteratorsize{T}(isz::T, ::T) = isz +and_iteratorsize(::HasLength, ::HasShape) = HasLength() +and_iteratorsize(::HasShape, ::HasLength) = HasLength() +and_iteratorsize(a, b) = SizeUnknown() + +and_iteratoreltype{T}(iel::T, ::T) = iel +and_iteratoreltype(a, b) = EltypeUnknown() + # enumerate immutable Enumerate{I} @@ -143,11 +159,6 @@ zip(a, b, c...) = Zip(a, zip(b, c...)) length(z::Zip) = _min_length(z.a, z.z, iteratorsize(z.a), iteratorsize(z.z)) size(z::Zip) = promote_shape(size(z.a), size(z.z)) indices(z::Zip) = promote_shape(indices(z.a), indices(z.z)) -tuple_type_cons{S}(::Type{S}, ::Type{Union{}}) = Union{} -function tuple_type_cons{S,T<:Tuple}(::Type{S}, ::Type{T}) - @_pure_meta - Tuple{S, T.parameters...} -end eltype{I,Z}(::Type{Zip{I,Z}}) = tuple_type_cons(eltype(I), eltype(Z)) @inline start(z::Zip) = tuple(start(z.a), start(z.z)) @inline function next(z::Zip, st) @@ -245,42 +256,6 @@ rest_iteratorsize(::IsInfinite) = IsInfinite() iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I)) -""" - head_and_tail(c, n) -> head, tail - -Returns `head`: the first `n` elements of `c`; -and `tail`: an iterator over the remaining elements. - -```jldoctest -julia> a = 1:10 -1:10 - -julia> b, c = Base.head_and_tail(a, 3) -([1,2,3],Base.Rest{UnitRange{Int64},Int64}(1:10,4)) - -julia> collect(c) -7-element Array{Any,1}: - 4 - 5 - 6 - 7 - 8 - 9 - 10 -``` -""" -function head_and_tail(c, n) - head = Vector{eltype(c)}(n) - s = start(c) - i = 0 - while i < n && !done(c, s) - i += 1 - head[i], s = next(c, s) - end - return resize!(head, i), rest(c, s) -end - - # Count -- infinite counting immutable Count{S<:Number} @@ -730,3 +705,5 @@ function next(itr::PartitionIterator, state) end return resize!(v, i), state end + +end diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 69b6ef1a9887b..c42b0832b6bdb 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -381,7 +381,7 @@ end # and ensure the value to set is either an AbstractArray or a Repeated scalar # before redispatching to the _unsafe_batchsetindex! _iterable(v::AbstractArray) = v -_iterable(v) = repeated(v) +_iterable(v) = IterTools.repeated(v) @inline function _setindex!{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, x, J::Vararg{Union{Real,AbstractArray,Colon},N}) @boundscheck checkbounds(A, J...) _unsafe_setindex!(l, A, x, J...) diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index 1f07d66fb98e3..9441584d3db8a 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -66,7 +66,7 @@ function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}}, vers = Array{Tuple{String,VersionNumber,VersionNumber}}(0) for (p,d) in deps, vn in keys(d) - lvns = VersionNumber[filter(vn2->(vn2>vn), keys(d))...] + lvns = VersionNumber[IterTools.filter(vn2->(vn2>vn), keys(d))...] nvn = isempty(lvns) ? typemax(VersionNumber) : minimum(lvns) push!(vers, (p,vn,nvn)) end diff --git a/base/pmap.jl b/base/pmap.jl index 9ea825067df32..f6c4aadf7eab1 100644 --- a/base/pmap.jl +++ b/base/pmap.jl @@ -213,6 +213,40 @@ function process_batch_errors!(p, f, results, on_error, retry_on, retry_n, retry nothing end +""" + head_and_tail(c, n) -> head, tail + +Returns `head`: the first `n` elements of `c`; +and `tail`: an iterator over the remaining elements. + +```jldoctest +julia> a = 1:10 +1:10 + +julia> b, c = Base.head_and_tail(a, 3) +([1,2,3],Base.IterTools.Rest{UnitRange{Int64},Int64}(1:10,4)) + +julia> collect(c) +7-element Array{Any,1}: + 4 + 5 + 6 + 7 + 8 + 9 + 10 +``` +""" +function head_and_tail(c, n) + head = Vector{eltype(c)}(n) + s = start(c) + i = 0 + while i < n && !done(c, s) + i += 1 + head[i], s = next(c, s) + end + return resize!(head, i), IterTools.rest(c, s) +end """ batchsplit(c; min_batch_count=1, max_batch_size=100) -> iterator @@ -231,13 +265,13 @@ function batchsplit(c; min_batch_count=1, max_batch_size=100) end # Split collection into batches, then peek at the first few batches - batches = partition(c, max_batch_size) + batches = IterTools.partition(c, max_batch_size) head, tail = head_and_tail(batches, min_batch_count) # If there are not enough batches, use a smaller batch size if length(head) < min_batch_count batch_size = max(1, div(sum(length, head), min_batch_count)) - return partition(collect(flatten(head)), batch_size) + return IterTools.partition(collect(flatten(head)), batch_size) end return flatten((head, tail)) diff --git a/base/promotion.jl b/base/promotion.jl index 4952126dc8cd8..e30349dfba865 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -226,7 +226,7 @@ if isdefined(Core, :Inference) end function _promote_op(op, R::ANY, S::ANY) F = typeof(a -> op(a...)) - G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}} + G = Tuple{Generator{IterTools.Zip2{Tuple{R},Tuple{S}},F}} return Core.Inference.return_type(first, G) end else diff --git a/base/sysimg.jl b/base/sysimg.jl index 20aca35a9b32e..56cf9cc399159 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -127,7 +127,9 @@ include("bitarray.jl") include("intset.jl") include("dict.jl") include("set.jl") -include("iterator.jl") +include("itertools.jl") +using .IterTools: zip, enumerate +using .IterTools: Flatten, product # for generators # Definition of StridedArray typealias StridedReshapedArray{T,N,A<:DenseArray} ReshapedArray{T,N,A} diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 489699536efe4..ae3b4d4b43146 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2152,7 +2152,7 @@ (car ranges) `(call (top product) ,@ranges))) (iter (if filt? - `(call (top Filter) + `(call (|.| (top IterTools) 'Filter) ,(func-for-generator-ranges (cadr (caddr e)) range-exprs) ,iter) iter))) diff --git a/test/choosetests.jl b/test/choosetests.jl index 03b91553e40d8..993e657cac790 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -20,7 +20,7 @@ function choosetests(choices = []) "dates", "dict", "hashing", "iobuffer", "staged", "offsetarray", "arrayops", "tuple", "reduce", "reducedim", "random", "abstractarray", "intfuncs", "simdloop", "vecelement", "blas", "sparse", - "bitarray", "copy", "math", "fastmath", "functional", + "bitarray", "copy", "math", "fastmath", "functional", "itertools", "operators", "path", "ccall", "parse", "loading", "bigint", "bigfloat", "sorting", "statistics", "spawn", "backtrace", "priorityqueue", "file", "read", "mmap", "version", "resolve", diff --git a/test/functional.jl b/test/functional.jl index 95331c878f159..cdd3f17ed0d3f 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -45,362 +45,8 @@ end # numbers @test size(collect(1)) == size(1) -# zip and filter iterators -# issue #4718 -@test collect(filter(x->x[1], zip([true, false, true, false],"abcd"))) == [(true,'a'),(true,'c')] - -let z = zip(1:2) - @test collect(z) == [(1,), (2,)] - # Issue #13979 - @test eltype(z) == Tuple{Int} -end - -let z = zip(1:2, 3:4) - @test collect(z) == [(1,3), (2,4)] - @test eltype(z) == Tuple{Int,Int} -end - -let z = zip(1:2, 3:4, 5:6) - @test collect(z) == [(1,3,5), (2,4,6)] - @test eltype(z) == Tuple{Int,Int,Int} -end - -@test eltype(Filter(isodd, 1:5)) == Int - -# typed `collect` -@test collect(Float64, Filter(isodd, [1,2,3,4]))[1] === 1.0 - @test isa(collect(Any, [1,2]), Vector{Any}) -# enumerate (issue #6284) -let b = IOBuffer("1\n2\n3\n"), a = [] - for (i,x) in enumerate(eachline(b)) - push!(a, (i,x)) - end - @test a == [(1,"1\n"),(2,"2\n"),(3,"3\n")] -end - -# zip eachline (issue #7369) -let zeb = IOBuffer("1\n2\n3\n4\n5\n"), - letters = ['a', 'b', 'c', 'd', 'e'], - res = [] - for (number, letter) in zip(eachline(zeb), letters) - push!(res, (parse(Int,strip(number)), letter)) - end - @test res == [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')] -end - -@test length(zip(cycle(1:3), 1:7)) == 7 -@test length(zip(cycle(1:3), 1:7, cycle(1:3))) == 7 -@test length(zip(1:3,Base.product(1:7,cycle(1:3)))) == 3 -@test length(zip(1:3,Base.product(1:7,cycle(1:3)),8)) == 1 - -# rest -# ---- -let s = "hello" - _, st = next(s, start(s)) - @test collect(rest(s, st)) == ['e','l','l','o'] -end - -@test_throws MethodError collect(rest(countfrom(1), 5)) - -# countfrom -# --------- - -let i = 0 - for j = countfrom(0, 2) - @test j == i*2 - i += 1 - i <= 10 || break - end -end - -# take -# ---- - -let t = take(0:2:8, 10), i = 0 - @test length(collect(t)) == 5 - - for j = t - @test j == i*2 - i += 1 - end - @test i == 5 -end - -let i = 0 - for j = take(0:2:100, 10) - @test j == i*2 - i += 1 - end - @test i == 10 -end - -@test length(take(1:3,typemax(Int))) == 3 -@test length(take(countfrom(1),3)) == 3 -@test length(take(1:6,3)) == 3 - -# drop -# ---- - -let i = 0 - for j = drop(0:2:10, 2) - @test j == (i+2)*2 - i += 1 - end - @test i == 4 -end - -@test length(drop(1:3,typemax(Int))) == 0 -@test Base.iteratorsize(drop(countfrom(1),3)) == Base.IsInfinite() -@test_throws MethodError length(drop(countfrom(1), 3)) - -# double take -# and take/drop canonicalization -# ----------- - -for xs in Any["abc", [1, 2, 3]] - @test take(take(xs, 2), 3) === take(xs, 2) - @test take(take(xs, 4), 2) === take(xs, 2) - @test drop(drop(xs, 1), 1) === drop(xs, 2) - @test take(drop(xs, 1), 1) === drop(take(xs, 2), 1) - @test take(drop(xs, 3), 0) === drop(take(xs, 2), 3) - @test isempty(drop(drop(xs, 2), 2)) - @test drop(take(drop(xs, 1), 2), 1) === take(drop(xs, 2), 1) - @test take(drop(take(xs, 3), 1), 1) === take(drop(xs, 1), 1) -end - -# cycle -# ----- - -let i = 0 - for j = cycle(0:3) - @test j == i % 4 - i += 1 - i <= 10 || break - end -end - -# repeated -# -------- - -let i = 0 - for j = repeated(1, 10) - @test j == 1 - i += 1 - end - @test i == 10 -end -let i = 0 - for j = repeated(1) - @test j == 1 - i += 1 - i <= 10 || break - end -end -@test eltype(repeated(0)) == Int -@test eltype(repeated(0, 5)) == Int -@test Base.iteratorsize(repeated(0)) == Base.IsInfinite() -@test Base.iteratorsize(repeated(0, 5)) == Base.HasLength() -@test Base.iteratoreltype(repeated(0)) == Base.HasEltype() -@test Base.iteratoreltype(repeated(0, 5)) == Base.HasEltype() -@test Base.iteratorsize(zip(repeated(0), repeated(0))) == Base.IsInfinite() - - -# product -# ------- - -# empty? -for itr in [Base.product(1:0), - Base.product(1:2, 1:0), - Base.product(1:0, 1:2), - Base.product(1:0, 1:1, 1:2), - Base.product(1:1, 1:0, 1:2), - Base.product(1:1, 1:2 ,1:0)] - @test isempty(itr) - @test isempty(collect(itr)) -end - -# collect a product - first iterators runs faster -@test collect(Base.product(1:2)) == [(i,) for i=1:2] -@test collect(Base.product(1:2, 3:4)) == [(i, j) for i=1:2, j=3:4] -@test collect(Base.product(1:2, 3:4, 5:6)) == [(i, j, k) for i=1:2, j=3:4, k=5:6] - -# iteration order -let - expected = [(1,3,5), (2,3,5), (1,4,5), (2,4,5), (1,3,6), (2,3,6), (1,4,6), (2,4,6)] - actual = Base.product(1:2, 3:4, 5:6) - for (exp, act) in zip(expected, actual) - @test exp == act - end -end - -# collect multidimensional array -let - a, b = 1:3, [4 6; - 5 7] - p = Base.product(a, b) - @test size(p) == (3, 2, 2) - @test length(p) == 12 - @test ndims(p) == 3 - @test eltype(p) == NTuple{2, Int} - cp = collect(p) - for i = 1:3 - @test cp[i, :, :] == [(i, 4) (i, 6); - (i, 5) (i, 7)] - end -end - -# with 1D inputs -let - a, b, c = 1:2, 1.0:10.0, Int32(1):Int32(0) - - # length - @test length(Base.product(a)) == 2 - @test length(Base.product(a, b)) == 20 - @test length(Base.product(a, b, c)) == 0 - - # size - @test size(Base.product(a)) == (2, ) - @test size(Base.product(a, b)) == (2, 10) - @test size(Base.product(a, b, c)) == (2, 10, 0) - - # eltype - @test eltype(Base.product(a)) == Tuple{Int} - @test eltype(Base.product(a, b)) == Tuple{Int, Float64} - @test eltype(Base.product(a, b, c)) == Tuple{Int, Float64, Int32} - - # ndims - @test ndims(Base.product(a)) == 1 - @test ndims(Base.product(a, b)) == 2 - @test ndims(Base.product(a, b, c)) == 3 -end - -# with multidimensional inputs -let - a, b, c = randn(4, 4), randn(3, 3, 3), randn(2, 2, 2, 2) - args = Any[(a,), - (a, a), - (a, b), - (a, a, a), - (a, b, c)] - sizes = Any[(4, 4), - (4, 4, 4, 4), - (4, 4, 3, 3, 3), - (4, 4, 4, 4, 4, 4), - (4, 4, 3, 3, 3, 2, 2, 2, 2)] - for (method, fun) in zip([size, ndims, length], [x->x, length, prod]) - for i in 1:length(args) - @test method(Base.product(args[i]...)) == method(collect(Base.product(args[i]...))) == fun(sizes[i]) - end - end -end - -# more tests on product with iterators of various type -let - iters = (1:2, - rand(2, 2, 2), - take(1:4, 2), - Base.product(1:2, 1:3), - Base.product(rand(2, 2), rand(1, 1, 1)) - ) - for method in [size, length, ndims, eltype] - for i = 1:length(iters) - args = iters[i] - @test method(Base.product(args...)) == method(collect(Base.product(args...))) - for j = 1:length(iters) - args = iters[i], iters[j] - @test method(Base.product(args...)) == method(collect(Base.product(args...))) - for k = 1:length(iters) - args = iters[i], iters[j], iters[k] - @test method(Base.product(args...)) == method(collect(Base.product(args...))) - end - end - end - end -end - -# product of finite length and infinite length iterators -let - a = 1:2 - b = countfrom(1) - ab = Base.product(a, b) - ba = Base.product(b, a) - abexp = [(1, 1), (2, 1), (1, 2), (2, 2), (1, 3), (2, 3)] - baexp = [(1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1)] - for (expected, actual) in zip([abexp, baexp], [ab, ba]) - for (i, el) in enumerate(actual) - @test el == expected[i] - i == length(expected) && break - end - @test_throws ArgumentError length(actual) - @test_throws ArgumentError size(actual) - @test_throws ArgumentError ndims(actual) - end - - # size infinite or unknown raises an error - for itr in Any[countfrom(1), Filter(i->0, 1:10)] - @test_throws ArgumentError length(Base.product(itr)) - @test_throws ArgumentError size(Base.product(itr)) - @test_throws ArgumentError ndims(Base.product(itr)) - end -end - -# iteratorsize trait business -let f1 = Filter(i->i>0, 1:10) - @test Base.iteratorsize(Base.product(f1)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(1:2, f1)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(f1, 1:2)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(f1, f1)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(f1, countfrom(1))) == Base.IsInfinite() - @test Base.iteratorsize(Base.product(countfrom(1), f1)) == Base.IsInfinite() -end -@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite() -@test Base.iteratorsize(Base.product(countfrom(2), countfrom(1))) == Base.IsInfinite() -@test Base.iteratorsize(Base.product(countfrom(1), 1:2)) == Base.IsInfinite() -@test Base.iteratorsize(Base.product(1:2)) == Base.HasShape() -@test Base.iteratorsize(Base.product(1:2, 1:2)) == Base.HasShape() -@test Base.iteratorsize(Base.product(take(1:2, 1), take(1:2, 1))) == Base.HasShape() -@test Base.iteratorsize(Base.product(take(1:2, 2))) == Base.HasLength() -@test Base.iteratorsize(Base.product([1 2; 3 4])) == Base.HasShape() - -# iteratoreltype trait business -let f1 = Filter(i->i>0, 1:10) - @test Base.iteratoreltype(Base.product(f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(1:2, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(f1, 1:2)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(f1, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(f1, countfrom(1))) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(countfrom(1), f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any -end -@test Base.iteratoreltype(Base.product(1:2, countfrom(1))) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(countfrom(1), 1:2)) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(1:2)) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(1:2, 1:2)) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(take(1:2, 1), take(1:2, 1))) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(take(1:2, 2))) == Base.HasEltype() -@test Base.iteratoreltype(Base.product([1 2; 3 4])) == Base.HasEltype() - -@test collect(Base.product(1:2,3:4)) == [(1,3) (1,4); (2,3) (2,4)] -@test isempty(collect(Base.product(1:0,1:2))) -@test length(Base.product(1:2,1:10,4:6)) == 60 -@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite() - -# flatten -# ------- - -import Base.flatten - -@test collect(flatten(Any[1:2, 4:5])) == Any[1,2,4,5] -@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[10:7, 10:9])])) == Any[1,2] -@test collect(flatten(Any[flatten(Any[1:2, 4:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,4,5,6,7,8,9] -@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,6,7,8,9] -@test collect(flatten(Any[2:1])) == Any[] -@test eltype(flatten(UnitRange{Int8}[1:2, 3:4])) == Int8 -@test_throws ArgumentError collect(flatten(Any[])) - -@test Base.iteratoreltype(Base.Flatten((i for i=1:2) for j=1:1)) == Base.EltypeUnknown() - # foreach let a = [] @@ -478,13 +124,13 @@ let gen = Base.Generator(+, 1:10, 1:10, 1:10) @test collect(gen) == collect(3:3:30) end -let gen = (x for x in 1:10 if x % 2 == 0), gen2 = Filter(x->x % 2 == 0, x for x in 1:10) +let gen = (x for x in 1:10 if x % 2 == 0), gen2 = IterTools.filter(x->x % 2 == 0, x for x in 1:10) @test collect(gen) == collect(gen2) @test collect(gen) == collect(2:2:10) end let gen = ((x,y) for x in 1:10, y in 1:10 if x % 2 == 0 && y % 2 == 0), - gen2 = Filter(x->x[1] % 2 == 0 && x[2] % 2 == 0, (x,y) for x in 1:10, y in 1:10) + gen2 = IterTools.filter(x->x[1] % 2 == 0 && x[2] % 2 == 0, (x,y) for x in 1:10, y in 1:10) @test collect(gen) == collect(gen2) end @@ -493,43 +139,3 @@ end @test [(i,j) for i=1:3 for j=1:i] == [(1,1), (2,1), (2,2), (3,1), (3,2), (3,3)] @test [(i,j) for i=1:3 for j=1:i if j>1] == [(2,2), (3,2), (3,3)] - -# partition(c, n) -let v = collect(Base.partition([1,2,3,4,5], 1)) - @test all(i->v[i][1] == i, v) -end - -let v = collect(Base.partition([1,2,3,4,5], 2)) - @test v[1] == [1,2] - @test v[2] == [3,4] - @test v[3] == [5] -end - -let v = collect(Base.partition(enumerate([1,2,3,4,5]), 3)) - @test v[1] == [(1,1),(2,2),(3,3)] - @test v[2] == [(4,4),(5,5)] -end - -for n in [5,6] - @test collect(Base.partition([1,2,3,4,5], n))[1] == [1,2,3,4,5] - @test collect(Base.partition(enumerate([1,2,3,4,5]), n))[1] == - [(1,1),(2,2),(3,3),(4,4),(5,5)] -end - - -@test join(map(x->string(x...), Base.partition("Hello World!", 5)), "|") == - "Hello| Worl|d!" - -let s = "Monkey 🙈🙊🙊" - tf = (n)->join(map(x->string(x...), Base.partition(s,n)), "|") - @test tf(10) == s - @test tf(9) == "Monkey 🙈🙊|🙊" - @test tf(8) == "Monkey 🙈|🙊🙊" - @test tf(7) == "Monkey |🙈🙊🙊" - @test tf(6) == "Monkey| 🙈🙊🙊" - @test tf(5) == "Monke|y 🙈🙊🙊" - @test tf(4) == "Monk|ey 🙈|🙊🙊" - @test tf(3) == "Mon|key| 🙈🙊|🙊" - @test tf(2) == "Mo|nk|ey| 🙈|🙊🙊" - @test tf(1) == "M|o|n|k|e|y| |🙈|🙊|🙊" -end diff --git a/test/itertools.jl b/test/itertools.jl new file mode 100644 index 0000000000000..f91b6a885392a --- /dev/null +++ b/test/itertools.jl @@ -0,0 +1,393 @@ +using Base.IterTools + +# zip and filter iterators +# issue #4718 +@test collect(IterTools.filter(x->x[1], zip([true, false, true, false],"abcd"))) == [(true,'a'),(true,'c')] + +let z = zip(1:2) + @test collect(z) == [(1,), (2,)] + # Issue #13979 + @test eltype(z) == Tuple{Int} +end + +let z = zip(1:2, 3:4) + @test collect(z) == [(1,3), (2,4)] + @test eltype(z) == Tuple{Int,Int} +end + +let z = zip(1:2, 3:4, 5:6) + @test collect(z) == [(1,3,5), (2,4,6)] + @test eltype(z) == Tuple{Int,Int,Int} +end + +@test eltype(IterTools.filter(isodd, 1:5)) == Int + +# typed `collect` +@test collect(Float64, IterTools.filter(isodd, [1,2,3,4]))[1] === 1.0 + +# enumerate (issue #6284) +let b = IOBuffer("1\n2\n3\n"), a = [] + for (i,x) in enumerate(eachline(b)) + push!(a, (i,x)) + end + @test a == [(1,"1\n"),(2,"2\n"),(3,"3\n")] +end + +# zip eachline (issue #7369) +let zeb = IOBuffer("1\n2\n3\n4\n5\n"), + letters = ['a', 'b', 'c', 'd', 'e'], + res = [] + for (number, letter) in zip(eachline(zeb), letters) + push!(res, (parse(Int,strip(number)), letter)) + end + @test res == [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')] +end + +@test length(zip(cycle(1:3), 1:7)) == 7 +@test length(zip(cycle(1:3), 1:7, cycle(1:3))) == 7 +@test length(zip(1:3,product(1:7,cycle(1:3)))) == 3 +@test length(zip(1:3,product(1:7,cycle(1:3)),8)) == 1 + +# rest +# ---- +let s = "hello" + _, st = next(s, start(s)) + @test collect(rest(s, st)) == ['e','l','l','o'] +end + +@test_throws MethodError collect(rest(countfrom(1), 5)) + +# countfrom +# --------- + +let i = 0 + for j = countfrom(0, 2) + @test j == i*2 + i += 1 + i <= 10 || break + end +end + +# take +# ---- + +let t = take(0:2:8, 10), i = 0 + @test length(collect(t)) == 5 + + for j = t + @test j == i*2 + i += 1 + end + @test i == 5 +end + +let i = 0 + for j = take(0:2:100, 10) + @test j == i*2 + i += 1 + end + @test i == 10 +end + +@test length(take(1:3,typemax(Int))) == 3 +@test length(take(countfrom(1),3)) == 3 +@test length(take(1:6,3)) == 3 + +# drop +# ---- + +let i = 0 + for j = drop(0:2:10, 2) + @test j == (i+2)*2 + i += 1 + end + @test i == 4 +end + +@test length(drop(1:3,typemax(Int))) == 0 +@test Base.iteratorsize(drop(countfrom(1),3)) == Base.IsInfinite() +@test_throws MethodError length(drop(countfrom(1), 3)) + +# double take +# and take/drop canonicalization +# ----------- + +for xs in Any["abc", [1, 2, 3]] + @test take(take(xs, 2), 3) === take(xs, 2) + @test take(take(xs, 4), 2) === take(xs, 2) + @test drop(drop(xs, 1), 1) === drop(xs, 2) + @test take(drop(xs, 1), 1) === drop(take(xs, 2), 1) + @test take(drop(xs, 3), 0) === drop(take(xs, 2), 3) + @test isempty(drop(drop(xs, 2), 2)) + @test drop(take(drop(xs, 1), 2), 1) === take(drop(xs, 2), 1) + @test take(drop(take(xs, 3), 1), 1) === take(drop(xs, 1), 1) +end + +# cycle +# ----- + +let i = 0 + for j = cycle(0:3) + @test j == i % 4 + i += 1 + i <= 10 || break + end +end + +# repeated +# -------- + +let i = 0 + for j = repeated(1, 10) + @test j == 1 + i += 1 + end + @test i == 10 +end +let i = 0 + for j = repeated(1) + @test j == 1 + i += 1 + i <= 10 || break + end +end +@test eltype(repeated(0)) == Int +@test eltype(repeated(0, 5)) == Int +@test Base.iteratorsize(repeated(0)) == Base.IsInfinite() +@test Base.iteratorsize(repeated(0, 5)) == Base.HasLength() +@test Base.iteratoreltype(repeated(0)) == Base.HasEltype() +@test Base.iteratoreltype(repeated(0, 5)) == Base.HasEltype() +@test Base.iteratorsize(zip(repeated(0), repeated(0))) == Base.IsInfinite() + + +# product +# ------- + +# empty? +for itr in [product(1:0), + product(1:2, 1:0), + product(1:0, 1:2), + product(1:0, 1:1, 1:2), + product(1:1, 1:0, 1:2), + product(1:1, 1:2 ,1:0)] + @test isempty(itr) + @test isempty(collect(itr)) +end + +# collect a product - first iterators runs faster +@test collect(product(1:2)) == [(i,) for i=1:2] +@test collect(product(1:2, 3:4)) == [(i, j) for i=1:2, j=3:4] +@test collect(product(1:2, 3:4, 5:6)) == [(i, j, k) for i=1:2, j=3:4, k=5:6] + +# iteration order +let + expected = [(1,3,5), (2,3,5), (1,4,5), (2,4,5), (1,3,6), (2,3,6), (1,4,6), (2,4,6)] + actual = product(1:2, 3:4, 5:6) + for (exp, act) in zip(expected, actual) + @test exp == act + end +end + +# collect multidimensional array +let + a, b = 1:3, [4 6; + 5 7] + p = product(a, b) + @test size(p) == (3, 2, 2) + @test length(p) == 12 + @test ndims(p) == 3 + @test eltype(p) == NTuple{2, Int} + cp = collect(p) + for i = 1:3 + @test cp[i, :, :] == [(i, 4) (i, 6); + (i, 5) (i, 7)] + end +end + +# with 1D inputs +let + a, b, c = 1:2, 1.0:10.0, Int32(1):Int32(0) + + # length + @test length(product(a)) == 2 + @test length(product(a, b)) == 20 + @test length(product(a, b, c)) == 0 + + # size + @test size(product(a)) == (2, ) + @test size(product(a, b)) == (2, 10) + @test size(product(a, b, c)) == (2, 10, 0) + + # eltype + @test eltype(product(a)) == Tuple{Int} + @test eltype(product(a, b)) == Tuple{Int, Float64} + @test eltype(product(a, b, c)) == Tuple{Int, Float64, Int32} + + # ndims + @test ndims(product(a)) == 1 + @test ndims(product(a, b)) == 2 + @test ndims(product(a, b, c)) == 3 +end + +# with multidimensional inputs +let + a, b, c = randn(4, 4), randn(3, 3, 3), randn(2, 2, 2, 2) + args = Any[(a,), + (a, a), + (a, b), + (a, a, a), + (a, b, c)] + sizes = Any[(4, 4), + (4, 4, 4, 4), + (4, 4, 3, 3, 3), + (4, 4, 4, 4, 4, 4), + (4, 4, 3, 3, 3, 2, 2, 2, 2)] + for (method, fun) in zip([size, ndims, length], [x->x, length, prod]) + for i in 1:length(args) + @test method(product(args[i]...)) == method(collect(product(args[i]...))) == fun(sizes[i]) + end + end +end + +# more tests on product with iterators of various type +let + iters = (1:2, + rand(2, 2, 2), + take(1:4, 2), + product(1:2, 1:3), + product(rand(2, 2), rand(1, 1, 1)) + ) + for method in [size, length, ndims, eltype] + for i = 1:length(iters) + args = iters[i] + @test method(product(args...)) == method(collect(product(args...))) + for j = 1:length(iters) + args = iters[i], iters[j] + @test method(product(args...)) == method(collect(product(args...))) + for k = 1:length(iters) + args = iters[i], iters[j], iters[k] + @test method(product(args...)) == method(collect(product(args...))) + end + end + end + end +end + +# product of finite length and infinite length iterators +let + a = 1:2 + b = countfrom(1) + ab = product(a, b) + ba = product(b, a) + abexp = [(1, 1), (2, 1), (1, 2), (2, 2), (1, 3), (2, 3)] + baexp = [(1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1)] + for (expected, actual) in zip([abexp, baexp], [ab, ba]) + for (i, el) in enumerate(actual) + @test el == expected[i] + i == length(expected) && break + end + @test_throws ArgumentError length(actual) + @test_throws ArgumentError size(actual) + @test_throws ArgumentError ndims(actual) + end + + # size infinite or unknown raises an error + for itr in Any[countfrom(1), IterTools.filter(i->0, 1:10)] + @test_throws ArgumentError length(product(itr)) + @test_throws ArgumentError size(product(itr)) + @test_throws ArgumentError ndims(product(itr)) + end +end + +# iteratorsize trait business +let f1 = IterTools.filter(i->i>0, 1:10) + @test Base.iteratorsize(product(f1)) == Base.SizeUnknown() + @test Base.iteratorsize(product(1:2, f1)) == Base.SizeUnknown() + @test Base.iteratorsize(product(f1, 1:2)) == Base.SizeUnknown() + @test Base.iteratorsize(product(f1, f1)) == Base.SizeUnknown() + @test Base.iteratorsize(product(f1, countfrom(1))) == Base.IsInfinite() + @test Base.iteratorsize(product(countfrom(1), f1)) == Base.IsInfinite() +end +@test Base.iteratorsize(product(1:2, countfrom(1))) == Base.IsInfinite() +@test Base.iteratorsize(product(countfrom(2), countfrom(1))) == Base.IsInfinite() +@test Base.iteratorsize(product(countfrom(1), 1:2)) == Base.IsInfinite() +@test Base.iteratorsize(product(1:2)) == Base.HasShape() +@test Base.iteratorsize(product(1:2, 1:2)) == Base.HasShape() +@test Base.iteratorsize(product(take(1:2, 1), take(1:2, 1))) == Base.HasShape() +@test Base.iteratorsize(product(take(1:2, 2))) == Base.HasLength() +@test Base.iteratorsize(product([1 2; 3 4])) == Base.HasShape() + +# iteratoreltype trait business +let f1 = IterTools.filter(i->i>0, 1:10) + @test Base.iteratoreltype(product(f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(1:2, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(f1, 1:2)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(f1, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(f1, countfrom(1))) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(countfrom(1), f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any +end +@test Base.iteratoreltype(product(1:2, countfrom(1))) == Base.HasEltype() +@test Base.iteratoreltype(product(countfrom(1), 1:2)) == Base.HasEltype() +@test Base.iteratoreltype(product(1:2)) == Base.HasEltype() +@test Base.iteratoreltype(product(1:2, 1:2)) == Base.HasEltype() +@test Base.iteratoreltype(product(take(1:2, 1), take(1:2, 1))) == Base.HasEltype() +@test Base.iteratoreltype(product(take(1:2, 2))) == Base.HasEltype() +@test Base.iteratoreltype(product([1 2; 3 4])) == Base.HasEltype() + +@test collect(product(1:2,3:4)) == [(1,3) (1,4); (2,3) (2,4)] +@test isempty(collect(product(1:0,1:2))) +@test length(product(1:2,1:10,4:6)) == 60 +@test Base.iteratorsize(product(1:2, countfrom(1))) == Base.IsInfinite() + +# flatten +# ------- + +@test collect(flatten(Any[1:2, 4:5])) == Any[1,2,4,5] +@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[10:7, 10:9])])) == Any[1,2] +@test collect(flatten(Any[flatten(Any[1:2, 4:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,4,5,6,7,8,9] +@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,6,7,8,9] +@test collect(flatten(Any[2:1])) == Any[] +@test eltype(flatten(UnitRange{Int8}[1:2, 3:4])) == Int8 +@test_throws ArgumentError collect(flatten(Any[])) + +@test Base.iteratoreltype(Base.Flatten((i for i=1:2) for j=1:1)) == Base.EltypeUnknown() + +# partition(c, n) +let v = collect(partition([1,2,3,4,5], 1)) + @test all(i->v[i][1] == i, v) +end + +let v = collect(partition([1,2,3,4,5], 2)) + @test v[1] == [1,2] + @test v[2] == [3,4] + @test v[3] == [5] +end + +let v = collect(partition(enumerate([1,2,3,4,5]), 3)) + @test v[1] == [(1,1),(2,2),(3,3)] + @test v[2] == [(4,4),(5,5)] +end + +for n in [5,6] + @test collect(partition([1,2,3,4,5], n))[1] == [1,2,3,4,5] + @test collect(partition(enumerate([1,2,3,4,5]), n))[1] == + [(1,1),(2,2),(3,3),(4,4),(5,5)] +end + + +@test join(map(x->string(x...), partition("Hello World!", 5)), "|") == + "Hello| Worl|d!" + +let s = "Monkey 🙈🙊🙊" + tf = (n)->join(map(x->string(x...), partition(s,n)), "|") + @test tf(10) == s + @test tf(9) == "Monkey 🙈🙊|🙊" + @test tf(8) == "Monkey 🙈|🙊🙊" + @test tf(7) == "Monkey |🙈🙊🙊" + @test tf(6) == "Monkey| 🙈🙊🙊" + @test tf(5) == "Monke|y 🙈🙊🙊" + @test tf(4) == "Monk|ey 🙈|🙊🙊" + @test tf(3) == "Mon|key| 🙈🙊|🙊" + @test tf(2) == "Mo|nk|ey| 🙈|🙊🙊" + @test tf(1) == "M|o|n|k|e|y| |🙈|🙊|🙊" +end