From 13b145f556b8669184ece79b2571b49f8807acd5 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Sat, 30 Jun 2018 15:49:45 -0700 Subject: [PATCH 1/9] Update REQUIRE for julia 0.7 --- REQUIRE | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/REQUIRE b/REQUIRE index 669fa41..a8be39a 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,3 @@ -julia 0.6 -NamedTuples 4.0.0 +julia 0.7- DataValues 0.0.1 TableTraits 0.0.1 From abbc817307067d93d3fd5a47688f0dbcfbe7403a Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 15:01:59 +0200 Subject: [PATCH 2/9] Update REQUIRE --- REQUIRE | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/REQUIRE b/REQUIRE index a8be39a..313934f 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,3 +1,5 @@ julia 0.7- -DataValues 0.0.1 -TableTraits 0.0.1 +DataValues +TableTraits 0.3.0 +IteratorInterfaceExtensions 0.1.0 +Missings 0.2.10 From 48fc595d44682a1ef15056e9b25e494dd98d2392 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 15:02:55 +0200 Subject: [PATCH 3/9] Update to julia 0.7 --- src/TableTraitsUtils.jl | 85 +++++++++++------------------- test/runtests.jl | 12 ++--- test/test_source_without_length.jl | 22 +++----- 3 files changed, 43 insertions(+), 76 deletions(-) diff --git a/src/TableTraitsUtils.jl b/src/TableTraitsUtils.jl index 7c5a87f..b182263 100644 --- a/src/TableTraitsUtils.jl +++ b/src/TableTraitsUtils.jl @@ -1,7 +1,7 @@ __precompile__() module TableTraitsUtils -using TableTraits, NamedTuples, DataValues +using IteratorInterfaceExtensions, TableTraits, DataValues, Missings export create_tableiterator, create_columns_from_iterabletable @@ -12,71 +12,46 @@ struct TableIterator{T, TS} end function create_tableiterator(columns, names::Vector{Symbol}) - col_expressions = Array{Expr,1}() - df_columns_tuple_type = Expr(:curly, :Tuple) - for i in 1:length(columns) - etype = eltype(columns[i]) - if etype <: Nullable - push!(col_expressions, Expr(:(::), names[i], DataValue{etype.parameters[1]})) + field_types = Type[] + for i in eltype.(columns) + if i >: Missing + push!(field_types, DataValue{Missings.T(i)}) else - push!(col_expressions, Expr(:(::), names[i], etype)) + push!(field_types, i) end - push!(df_columns_tuple_type.args, typeof(columns[i])) end - t_expr = NamedTuples.make_tuple(col_expressions) - - t2 = :(TableIterator{Float64,Float64}) - t2.args[2] = t_expr - t2.args[3] = df_columns_tuple_type - - t = eval(t2) - - e_df = t((columns...)) - - return e_df + return TableIterator{NamedTuple{(names...,), Tuple{field_types...}}, Tuple{typeof.(columns)...}}((columns...,)) end function Base.length(iter::TableIterator{T,TS}) where {T,TS} return length(iter.columns[1]) end -function Base.eltype(iter::TableIterator{T,TS}) where {T,TS} - return T -end - Base.eltype(::Type{TableIterator{T,TS}}) where {T,TS} = T -function Base.start(iter::TableIterator{T,TS}) where {T,TS} - return 1 -end - -@generated function Base.next(iter::TableIterator{T,TS}, state) where {T,TS} - constructor_call = Expr(:call, :($T)) - for (i,t) in enumerate(T.parameters) - if eltype(iter.parameters[2].parameters[i]) <: Nullable - push!(constructor_call.args, :(DataValue(columns[$i][i]))) +@generated function Base.iterate(iter::TableIterator{T,TS}, state=1) where {T,TS} + columns = map(1:length(TS.parameters)) do i + if fieldtype(T,i) <: DataValue && eltype(TS.parameters[i]) >: Missing + return :($(fieldtype(T,i))(iter.columns[$i][state])) else - push!(constructor_call.args, :(columns[$i][i])) + return :(iter.columns[$i][state]) end end - - quote - i = state - columns = iter.columns - a = $constructor_call - return a, state+1 + # [:(iter.columns[$i][state]) for i in 1:length(TS.parameters)] + return quote + if state > length(iter) + return nothing + else + return $(T)(($(columns...),)), state+1 + end end end -function Base.done(iter::TableIterator{T,TS}, state) where {T,TS} - return state>length(iter.columns[1]) -end - # Sink @generated function _fill_cols_without_length(columns, enumerable) push_exprs = Expr(:block) - for i in find(collect(columns.types) .!= Void) + for i in findall(collect(columns.types) .!= Nothing) ex = :( push!(columns[$i], i[$i]) ) push!(push_exprs.args, ex) end @@ -90,7 +65,7 @@ end @generated function _fill_cols_with_length(columns, enumerable) push_exprs = Expr(:block) - for col_idx in find(collect(columns.types) .!= Void) + for col_idx in findall(collect(columns.types) .!= Nothing) ex = :( columns[$col_idx][i] = v[$col_idx] ) push!(push_exprs.args, ex) end @@ -104,13 +79,13 @@ end function _default_array_factory(t,rows) if isa(t, TypeVar) - return Array{Any}(rows) + return Array{Any}(undef, rows) else - return Array{t}(rows) + return Array{t}(undef, rows) end end -function create_columns_from_iterabletable(source, sel_cols = :all; array_factory::Function=_default_array_factory) +function create_columns_from_iterabletable(source; sel_cols=:all, array_factory=_default_array_factory) iter = getiterator(source) T = eltype(iter) @@ -118,10 +93,10 @@ function create_columns_from_iterabletable(source, sel_cols = :all; array_factor error("Can only collect a NamedTuple iterator.") end - column_types = TableTraits.column_types(iter) - column_names = TableTraits.column_names(iter) + column_types = collect(T.parameters[2].parameters) + column_names = collect(T.parameters[1]) - rows = Base.iteratorsize(typeof(iter))==Base.HasLength() ? length(iter) : 0 + rows = Base.IteratorSize(typeof(iter))==Base.HasLength() ? length(iter) : 0 columns = [] for (i, t) in enumerate(column_types) @@ -132,10 +107,10 @@ function create_columns_from_iterabletable(source, sel_cols = :all; array_factor end end - if Base.iteratorsize(typeof(iter))==Base.HasLength() - _fill_cols_with_length((columns...), iter) + if Base.IteratorSize(typeof(iter))==Base.HasLength() + _fill_cols_with_length((columns...,), iter) else - _fill_cols_without_length((columns...), iter) + _fill_cols_without_length((columns...,), iter) end if sel_cols == :all diff --git a/test/runtests.jl b/test/runtests.jl index 431e37d..3f1ebc0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ using TableTraitsUtils using DataValues -using Base.Test +using Test include("test_source_without_length.jl") @@ -13,9 +13,9 @@ it = TableTraitsUtils.create_tableiterator(columns, names) columns2, names2 = TableTraitsUtils.create_columns_from_iterabletable(it) -columns3, names3 = TableTraitsUtils.create_columns_from_iterabletable(it, :all) +columns3, names3 = TableTraitsUtils.create_columns_from_iterabletable(it, sel_cols=:all) -columns23, names23 = TableTraitsUtils.create_columns_from_iterabletable(it, [2,3]) +columns23, names23 = TableTraitsUtils.create_columns_from_iterabletable(it, sel_cols=[2,3]) @test columns[1] == columns2[1] == columns3[1] @test columns[2] == columns2[2] == columns3[2] @@ -35,16 +35,16 @@ columns4, names4 = TableTraitsUtils.create_columns_from_iterabletable(it2) @test columns4[2] == [1.,2.] @test names4 == [:a, :b] -columns5, names5 = TableTraitsUtils.create_columns_from_iterabletable(it2, :all) +columns5, names5 = TableTraitsUtils.create_columns_from_iterabletable(it2, sel_cols=:all) @test columns5[1] == [1,2] @test columns5[2] == [1.,2.] @test names5 == [:a, :b] -columns6, names6 = TableTraitsUtils.create_columns_from_iterabletable(it2, [2]) +columns6, names6 = TableTraitsUtils.create_columns_from_iterabletable(it2, sel_cols=[2]) @test columns6[1] == [1.,2.] @test names6 == [:b] -columns_with_nulls = (Nullable{Int}[Nullable(3), Nullable(2), Nullable{Int}()], [2.,5.,9.], Nullable{String}[Nullable("a"), Nullable{String}(), Nullable("b")]) +columns_with_nulls = (Union{Int,Missing}[3, 2, missing], Float64[2.,5.,9.], Union{String,Missing}["a", missing, "b"]) it3 = TableTraitsUtils.create_tableiterator(columns_with_nulls, names) columns7, names7 = TableTraitsUtils.create_columns_from_iterabletable(it3) diff --git a/test/test_source_without_length.jl b/test/test_source_without_length.jl index e069e6f..5750d42 100644 --- a/test/test_source_without_length.jl +++ b/test/test_source_without_length.jl @@ -1,26 +1,18 @@ -using NamedTuples - struct TestSourceWithoutLength end function Base.eltype(iter::TestSourceWithoutLength) - return @NT(a::Int, b::Float64) + return NamedTuple{(:a, :b), Tuple{Int, Float64}} end -Base.iteratorsize(::Type{T}) where {T <: TestSourceWithoutLength} = Base.SizeUnknown() - -function Base.start(iter::TestSourceWithoutLength) - return 1 -end +Base.IteratorSize(::Type{T}) where {T <: TestSourceWithoutLength} = Base.SizeUnknown() -function Base.next(iter::TestSourceWithoutLength, state) +function Base.iterate(iter::TestSourceWithoutLength, state=1) if state==1 - return @NT(a=1, b=1.), 2 + return (a=1, b=1.), 2 elseif state==2 - return @NT(a=2, b=2.), 3 + return (a=2, b=2.), 3 + else + return nothing end end - -function Base.done(iter::TestSourceWithoutLength, state) - return state>2 -end \ No newline at end of file From 9651742669fbbb2a60b896ae674706ad409cfe99 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 16:48:47 +0200 Subject: [PATCH 4/9] Remove unnecessary code --- src/TableTraitsUtils.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/TableTraitsUtils.jl b/src/TableTraitsUtils.jl index b182263..9357909 100644 --- a/src/TableTraitsUtils.jl +++ b/src/TableTraitsUtils.jl @@ -37,7 +37,6 @@ Base.eltype(::Type{TableIterator{T,TS}}) where {T,TS} = T return :(iter.columns[$i][state]) end end - # [:(iter.columns[$i][state]) for i in 1:length(TS.parameters)] return quote if state > length(iter) return nothing From 875f6b49ed06014a4d3c5829fa1e56dbc4a22ffe Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 16:48:59 +0200 Subject: [PATCH 5/9] Update CI config --- .travis.yml | 3 ++- appveyor.yml | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5bf074..43f13a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ os: - linux - osx julia: - - 0.6 + - 0.7 + - nightly notifications: email: false git: diff --git a/appveyor.yml b/appveyor.yml index 5a0ce3c..7b50e4d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,9 @@ environment: matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.7/julia-0.7-latest-win32.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7-latest-win64.exe" + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" ## uncomment the following lines to allow failures on nightly julia ## (tests will run but not make your overall status red) From d9834af1d72cb01ef90a9e563cedfe3ffed94a2d Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 16:49:07 +0200 Subject: [PATCH 6/9] Update LICENSE --- LICENSE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.md b/LICENSE.md index 241b2a6..eb61630 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The TableTraitsUtils.jl package is licensed under the MIT "Expat" License: -> Copyright (c) 2017: David Anthoff. +> Copyright (c) 2017-2018: David Anthoff. > > > Permission is hereby granted, free of charge, to any person obtaining a copy From cf392eb16a1612a4c4afece263c9ddd909a63bfb Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 16:49:13 +0200 Subject: [PATCH 7/9] Update NEWS --- NEWS.md | 3 +++ REQUIRE | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 75532be..3553e41 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,6 @@ +# TableTraitsUtils.jl v0.2.0 Release Notes +* Drop julia 0.6 support, add julia 0.7 support + # TableTraitsUtils.jl v0.1.3 Release Notes * Add missing eltype method diff --git a/REQUIRE b/REQUIRE index 313934f..bb9a7b7 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,5 @@ julia 0.7- -DataValues +DataValues 0.4.0 TableTraits 0.3.0 IteratorInterfaceExtensions 0.1.0 Missings 0.2.10 From a34a88d3832d7dee9b4a523bde6f7cde2f22af1e Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Tue, 10 Jul 2018 16:49:31 +0200 Subject: [PATCH 8/9] Update REQUIRE --- REQUIRE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIRE b/REQUIRE index bb9a7b7..ff6541f 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,5 @@ julia 0.7- -DataValues 0.4.0 +DataValues 0.4.1 TableTraits 0.3.0 IteratorInterfaceExtensions 0.1.0 Missings 0.2.10 From 8b0ab107902804aebadd37b7800c9d7daf049fd5 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 11 Jul 2018 00:00:41 +0200 Subject: [PATCH 9/9] Change API for Missing vs DataValue materialization --- src/TableTraitsUtils.jl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/TableTraitsUtils.jl b/src/TableTraitsUtils.jl index 9357909..b2f1dd2 100644 --- a/src/TableTraitsUtils.jl +++ b/src/TableTraitsUtils.jl @@ -76,15 +76,7 @@ end end end -function _default_array_factory(t,rows) - if isa(t, TypeVar) - return Array{Any}(undef, rows) - else - return Array{t}(undef, rows) - end -end - -function create_columns_from_iterabletable(source; sel_cols=:all, array_factory=_default_array_factory) +function create_columns_from_iterabletable(source; sel_cols=:all, na_representation=:datavalue) iter = getiterator(source) T = eltype(iter) @@ -92,6 +84,18 @@ function create_columns_from_iterabletable(source; sel_cols=:all, array_factory= error("Can only collect a NamedTuple iterator.") end + array_factory = if na_representation==:datavalue + (t,rows) -> Array{t}(undef, rows) + elseif na_representation==:missing + (t,rows) -> begin + if t <: DataValue + return Array{Union{eltype(t),Missing}}(undef, rows) + else + return Array{t}(undef, rows) + end + end + end + column_types = collect(T.parameters[2].parameters) column_names = collect(T.parameters[1])