diff --git a/.travis.yml b/.travis.yml index b5bf074..43f13a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ os: - linux - osx julia: - - 0.6 + - 0.7 + - nightly notifications: email: false git: diff --git a/LICENSE.md b/LICENSE.md index 241b2a6..eb61630 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The TableTraitsUtils.jl package is licensed under the MIT "Expat" License: -> Copyright (c) 2017: David Anthoff. +> Copyright (c) 2017-2018: David Anthoff. > > > Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/NEWS.md b/NEWS.md index 75532be..3553e41 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +# TableTraitsUtils.jl v0.2.0 Release Notes +* Drop julia 0.6 support, add julia 0.7 support + # TableTraitsUtils.jl v0.1.3 Release Notes * Add missing eltype method diff --git a/REQUIRE b/REQUIRE index 669fa41..ff6541f 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,5 @@ -julia 0.6 -NamedTuples 4.0.0 -DataValues 0.0.1 -TableTraits 0.0.1 +julia 0.7- +DataValues 0.4.1 +TableTraits 0.3.0 +IteratorInterfaceExtensions 0.1.0 +Missings 0.2.10 diff --git a/appveyor.yml b/appveyor.yml index 5a0ce3c..7b50e4d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,9 @@ environment: matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.7/julia-0.7-latest-win32.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7-latest-win64.exe" + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" ## uncomment the following lines to allow failures on nightly julia ## (tests will run but not make your overall status red) diff --git a/src/TableTraitsUtils.jl b/src/TableTraitsUtils.jl index 7c5a87f..b2f1dd2 100644 --- a/src/TableTraitsUtils.jl +++ b/src/TableTraitsUtils.jl @@ -1,7 +1,7 @@ __precompile__() module TableTraitsUtils -using TableTraits, NamedTuples, DataValues +using IteratorInterfaceExtensions, TableTraits, DataValues, Missings export create_tableiterator, create_columns_from_iterabletable @@ -12,71 +12,45 @@ struct TableIterator{T, TS} end function create_tableiterator(columns, names::Vector{Symbol}) - col_expressions = Array{Expr,1}() - df_columns_tuple_type = Expr(:curly, :Tuple) - for i in 1:length(columns) - etype = eltype(columns[i]) - if etype <: Nullable - push!(col_expressions, Expr(:(::), names[i], DataValue{etype.parameters[1]})) + field_types = Type[] + for i in eltype.(columns) + if i >: Missing + push!(field_types, DataValue{Missings.T(i)}) else - push!(col_expressions, Expr(:(::), names[i], etype)) + push!(field_types, i) end - push!(df_columns_tuple_type.args, typeof(columns[i])) end - t_expr = NamedTuples.make_tuple(col_expressions) - - t2 = :(TableIterator{Float64,Float64}) - t2.args[2] = t_expr - t2.args[3] = df_columns_tuple_type - - t = eval(t2) - - e_df = t((columns...)) - - return e_df + return TableIterator{NamedTuple{(names...,), Tuple{field_types...}}, Tuple{typeof.(columns)...}}((columns...,)) end function Base.length(iter::TableIterator{T,TS}) where {T,TS} return length(iter.columns[1]) end -function Base.eltype(iter::TableIterator{T,TS}) where {T,TS} - return T -end - Base.eltype(::Type{TableIterator{T,TS}}) where {T,TS} = T -function Base.start(iter::TableIterator{T,TS}) where {T,TS} - return 1 -end - -@generated function Base.next(iter::TableIterator{T,TS}, state) where {T,TS} - constructor_call = Expr(:call, :($T)) - for (i,t) in enumerate(T.parameters) - if eltype(iter.parameters[2].parameters[i]) <: Nullable - push!(constructor_call.args, :(DataValue(columns[$i][i]))) +@generated function Base.iterate(iter::TableIterator{T,TS}, state=1) where {T,TS} + columns = map(1:length(TS.parameters)) do i + if fieldtype(T,i) <: DataValue && eltype(TS.parameters[i]) >: Missing + return :($(fieldtype(T,i))(iter.columns[$i][state])) else - push!(constructor_call.args, :(columns[$i][i])) + return :(iter.columns[$i][state]) end end - - quote - i = state - columns = iter.columns - a = $constructor_call - return a, state+1 + return quote + if state > length(iter) + return nothing + else + return $(T)(($(columns...),)), state+1 + end end end -function Base.done(iter::TableIterator{T,TS}, state) where {T,TS} - return state>length(iter.columns[1]) -end - # Sink @generated function _fill_cols_without_length(columns, enumerable) push_exprs = Expr(:block) - for i in find(collect(columns.types) .!= Void) + for i in findall(collect(columns.types) .!= Nothing) ex = :( push!(columns[$i], i[$i]) ) push!(push_exprs.args, ex) end @@ -90,7 +64,7 @@ end @generated function _fill_cols_with_length(columns, enumerable) push_exprs = Expr(:block) - for col_idx in find(collect(columns.types) .!= Void) + for col_idx in findall(collect(columns.types) .!= Nothing) ex = :( columns[$col_idx][i] = v[$col_idx] ) push!(push_exprs.args, ex) end @@ -102,15 +76,7 @@ end end end -function _default_array_factory(t,rows) - if isa(t, TypeVar) - return Array{Any}(rows) - else - return Array{t}(rows) - end -end - -function create_columns_from_iterabletable(source, sel_cols = :all; array_factory::Function=_default_array_factory) +function create_columns_from_iterabletable(source; sel_cols=:all, na_representation=:datavalue) iter = getiterator(source) T = eltype(iter) @@ -118,10 +84,22 @@ function create_columns_from_iterabletable(source, sel_cols = :all; array_factor error("Can only collect a NamedTuple iterator.") end - column_types = TableTraits.column_types(iter) - column_names = TableTraits.column_names(iter) + array_factory = if na_representation==:datavalue + (t,rows) -> Array{t}(undef, rows) + elseif na_representation==:missing + (t,rows) -> begin + if t <: DataValue + return Array{Union{eltype(t),Missing}}(undef, rows) + else + return Array{t}(undef, rows) + end + end + end + + column_types = collect(T.parameters[2].parameters) + column_names = collect(T.parameters[1]) - rows = Base.iteratorsize(typeof(iter))==Base.HasLength() ? length(iter) : 0 + rows = Base.IteratorSize(typeof(iter))==Base.HasLength() ? length(iter) : 0 columns = [] for (i, t) in enumerate(column_types) @@ -132,10 +110,10 @@ function create_columns_from_iterabletable(source, sel_cols = :all; array_factor end end - if Base.iteratorsize(typeof(iter))==Base.HasLength() - _fill_cols_with_length((columns...), iter) + if Base.IteratorSize(typeof(iter))==Base.HasLength() + _fill_cols_with_length((columns...,), iter) else - _fill_cols_without_length((columns...), iter) + _fill_cols_without_length((columns...,), iter) end if sel_cols == :all diff --git a/test/runtests.jl b/test/runtests.jl index 431e37d..3f1ebc0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ using TableTraitsUtils using DataValues -using Base.Test +using Test include("test_source_without_length.jl") @@ -13,9 +13,9 @@ it = TableTraitsUtils.create_tableiterator(columns, names) columns2, names2 = TableTraitsUtils.create_columns_from_iterabletable(it) -columns3, names3 = TableTraitsUtils.create_columns_from_iterabletable(it, :all) +columns3, names3 = TableTraitsUtils.create_columns_from_iterabletable(it, sel_cols=:all) -columns23, names23 = TableTraitsUtils.create_columns_from_iterabletable(it, [2,3]) +columns23, names23 = TableTraitsUtils.create_columns_from_iterabletable(it, sel_cols=[2,3]) @test columns[1] == columns2[1] == columns3[1] @test columns[2] == columns2[2] == columns3[2] @@ -35,16 +35,16 @@ columns4, names4 = TableTraitsUtils.create_columns_from_iterabletable(it2) @test columns4[2] == [1.,2.] @test names4 == [:a, :b] -columns5, names5 = TableTraitsUtils.create_columns_from_iterabletable(it2, :all) +columns5, names5 = TableTraitsUtils.create_columns_from_iterabletable(it2, sel_cols=:all) @test columns5[1] == [1,2] @test columns5[2] == [1.,2.] @test names5 == [:a, :b] -columns6, names6 = TableTraitsUtils.create_columns_from_iterabletable(it2, [2]) +columns6, names6 = TableTraitsUtils.create_columns_from_iterabletable(it2, sel_cols=[2]) @test columns6[1] == [1.,2.] @test names6 == [:b] -columns_with_nulls = (Nullable{Int}[Nullable(3), Nullable(2), Nullable{Int}()], [2.,5.,9.], Nullable{String}[Nullable("a"), Nullable{String}(), Nullable("b")]) +columns_with_nulls = (Union{Int,Missing}[3, 2, missing], Float64[2.,5.,9.], Union{String,Missing}["a", missing, "b"]) it3 = TableTraitsUtils.create_tableiterator(columns_with_nulls, names) columns7, names7 = TableTraitsUtils.create_columns_from_iterabletable(it3) diff --git a/test/test_source_without_length.jl b/test/test_source_without_length.jl index e069e6f..5750d42 100644 --- a/test/test_source_without_length.jl +++ b/test/test_source_without_length.jl @@ -1,26 +1,18 @@ -using NamedTuples - struct TestSourceWithoutLength end function Base.eltype(iter::TestSourceWithoutLength) - return @NT(a::Int, b::Float64) + return NamedTuple{(:a, :b), Tuple{Int, Float64}} end -Base.iteratorsize(::Type{T}) where {T <: TestSourceWithoutLength} = Base.SizeUnknown() - -function Base.start(iter::TestSourceWithoutLength) - return 1 -end +Base.IteratorSize(::Type{T}) where {T <: TestSourceWithoutLength} = Base.SizeUnknown() -function Base.next(iter::TestSourceWithoutLength, state) +function Base.iterate(iter::TestSourceWithoutLength, state=1) if state==1 - return @NT(a=1, b=1.), 2 + return (a=1, b=1.), 2 elseif state==2 - return @NT(a=2, b=2.), 3 + return (a=2, b=2.), 3 + else + return nothing end end - -function Base.done(iter::TestSourceWithoutLength, state) - return state>2 -end \ No newline at end of file