Merge pull request #6 from davidanthoff/julia-0.7

julia 0.7

.travis.yml

@@ -4,7 +4,8 @@ os:
   - linux
   - osx
 julia:
-  - 0.6
+  - 0.7
+  - nightly
 notifications:
   email: false
 git:

LICENSE.md

@@ -1,6 +1,6 @@
 The TableTraitsUtils.jl package is licensed under the MIT "Expat" License:
 
-> Copyright (c) 2017: David Anthoff.
+> Copyright (c) 2017-2018: David Anthoff.
 > 
 > 
 > Permission is hereby granted, free of charge, to any person obtaining a copy

NEWS.md

@@ -1,3 +1,6 @@
+# TableTraitsUtils.jl v0.2.0 Release Notes
+* Drop julia 0.6 support, add julia 0.7 support
+
 # TableTraitsUtils.jl v0.1.3 Release Notes
 * Add missing eltype method
 

REQUIRE

@@ -1,4 +1,5 @@
-julia 0.6
-NamedTuples 4.0.0
-DataValues 0.0.1
-TableTraits 0.0.1
+julia 0.7-
+DataValues 0.4.1
+TableTraits 0.3.0
+IteratorInterfaceExtensions 0.1.0
+Missings 0.2.10

appveyor.yml

@@ -1,7 +1,9 @@
 environment:
   matrix:
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
+  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.7/julia-0.7-latest-win32.exe"
+  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7-latest-win64.exe"
+  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
+  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
 
 ## uncomment the following lines to allow failures on nightly julia
 ## (tests will run but not make your overall status red)

src/TableTraitsUtils.jl

@@ -1,7 +1,7 @@
 __precompile__()
 module TableTraitsUtils
 
-using TableTraits, NamedTuples, DataValues
+using IteratorInterfaceExtensions, TableTraits, DataValues, Missings
 
 export create_tableiterator, create_columns_from_iterabletable
 
@@ -12,71 +12,45 @@ struct TableIterator{T, TS}
 end
 
 function create_tableiterator(columns, names::Vector{Symbol})
-    col_expressions = Array{Expr,1}()
-    df_columns_tuple_type = Expr(:curly, :Tuple)
-    for i in 1:length(columns)
-        etype = eltype(columns[i])
-        if etype <: Nullable
-            push!(col_expressions, Expr(:(::), names[i], DataValue{etype.parameters[1]}))
+    field_types = Type[]
+    for i in eltype.(columns)
+        if i >: Missing
+            push!(field_types, DataValue{Missings.T(i)})
         else
-            push!(col_expressions, Expr(:(::), names[i], etype))
+            push!(field_types, i)
         end
-        push!(df_columns_tuple_type.args, typeof(columns[i]))
     end
-    t_expr = NamedTuples.make_tuple(col_expressions)
-
-    t2 = :(TableIterator{Float64,Float64})
-    t2.args[2] = t_expr
-    t2.args[3] = df_columns_tuple_type
-
-    t = eval(t2)
-
-    e_df = t((columns...))
-
-    return e_df
+    return TableIterator{NamedTuple{(names...,), Tuple{field_types...}}, Tuple{typeof.(columns)...}}((columns...,))
 end
 
 function Base.length(iter::TableIterator{T,TS}) where {T,TS}
     return length(iter.columns[1])
 end
 
-function Base.eltype(iter::TableIterator{T,TS}) where {T,TS}
-    return T
-end
-
 Base.eltype(::Type{TableIterator{T,TS}}) where {T,TS} = T
 
-function Base.start(iter::TableIterator{T,TS}) where {T,TS}
-    return 1
-end
-
-@generated function Base.next(iter::TableIterator{T,TS}, state) where {T,TS}
-    constructor_call = Expr(:call, :($T))
-    for (i,t) in enumerate(T.parameters)
-        if eltype(iter.parameters[2].parameters[i]) <: Nullable
-            push!(constructor_call.args, :(DataValue(columns[$i][i])))
+@generated function Base.iterate(iter::TableIterator{T,TS}, state=1) where {T,TS}
+    columns = map(1:length(TS.parameters)) do i
+        if fieldtype(T,i) <: DataValue && eltype(TS.parameters[i]) >: Missing
+            return :($(fieldtype(T,i))(iter.columns[$i][state]))
         else
-            push!(constructor_call.args, :(columns[$i][i]))
+            return :(iter.columns[$i][state])
         end
     end
-
-    quote
-        i = state
-        columns = iter.columns
-        a = $constructor_call
-        return a, state+1
+    return quote
+        if state > length(iter)
+            return nothing
+        else            
+            return $(T)(($(columns...),)), state+1
+        end
     end
 end
 
-function Base.done(iter::TableIterator{T,TS}, state) where {T,TS}
-    return state>length(iter.columns[1])
-end
-
 # Sink
 
 @generated function _fill_cols_without_length(columns, enumerable)
     push_exprs = Expr(:block)
-    for i in find(collect(columns.types) .!= Void)
+    for i in findall(collect(columns.types) .!= Nothing)
         ex = :( push!(columns[$i], i[$i]) )
         push!(push_exprs.args, ex)
     end
@@ -90,7 +64,7 @@ end
 
 @generated function _fill_cols_with_length(columns, enumerable)
     push_exprs = Expr(:block)
-    for col_idx in find(collect(columns.types) .!= Void)
+    for col_idx in findall(collect(columns.types) .!= Nothing)
         ex = :( columns[$col_idx][i] = v[$col_idx] )
         push!(push_exprs.args, ex)
     end
@@ -102,26 +76,30 @@ end
     end
 end
 
-function _default_array_factory(t,rows)
-    if isa(t, TypeVar)
-        return Array{Any}(rows)
-    else
-        return Array{t}(rows)
-    end
-end
-
-function create_columns_from_iterabletable(source, sel_cols = :all; array_factory::Function=_default_array_factory)
+function create_columns_from_iterabletable(source; sel_cols=:all, na_representation=:datavalue)
     iter = getiterator(source)
 
     T = eltype(iter)
     if !(T<:NamedTuple)
         error("Can only collect a NamedTuple iterator.")
     end
 
-    column_types = TableTraits.column_types(iter)
-    column_names = TableTraits.column_names(iter)
+    array_factory = if na_representation==:datavalue
+        (t,rows) -> Array{t}(undef, rows)
+    elseif na_representation==:missing
+        (t,rows) -> begin
+            if t <: DataValue
+                return Array{Union{eltype(t),Missing}}(undef, rows)
+            else
+                return Array{t}(undef, rows)
+            end
+        end
+    end
+
+    column_types = collect(T.parameters[2].parameters)
+    column_names = collect(T.parameters[1])
 
-    rows = Base.iteratorsize(typeof(iter))==Base.HasLength() ? length(iter) : 0
+    rows = Base.IteratorSize(typeof(iter))==Base.HasLength() ? length(iter) : 0
 
     columns = []
     for (i, t) in enumerate(column_types)
@@ -132,10 +110,10 @@ function create_columns_from_iterabletable(source, sel_cols = :all; array_factor
         end
     end
 
-    if Base.iteratorsize(typeof(iter))==Base.HasLength()
-        _fill_cols_with_length((columns...), iter)
+    if Base.IteratorSize(typeof(iter))==Base.HasLength()
+        _fill_cols_with_length((columns...,), iter)
     else
-        _fill_cols_without_length((columns...), iter)
+        _fill_cols_without_length((columns...,), iter)
     end
 
     if sel_cols == :all

test/runtests.jl

@@ -1,6 +1,6 @@
 using TableTraitsUtils
 using DataValues
-using Base.Test
+using Test
 
 include("test_source_without_length.jl")
 
@@ -13,9 +13,9 @@ it = TableTraitsUtils.create_tableiterator(columns, names)
 
 columns2, names2 = TableTraitsUtils.create_columns_from_iterabletable(it)
 
-columns3, names3 = TableTraitsUtils.create_columns_from_iterabletable(it, :all)
+columns3, names3 = TableTraitsUtils.create_columns_from_iterabletable(it, sel_cols=:all)
 
-columns23, names23 = TableTraitsUtils.create_columns_from_iterabletable(it, [2,3])
+columns23, names23 = TableTraitsUtils.create_columns_from_iterabletable(it, sel_cols=[2,3])
 
 @test columns[1] == columns2[1] == columns3[1]
 @test columns[2] == columns2[2] == columns3[2]
@@ -35,16 +35,16 @@ columns4, names4 = TableTraitsUtils.create_columns_from_iterabletable(it2)
 @test columns4[2] == [1.,2.]
 @test names4 == [:a, :b]
 
-columns5, names5 = TableTraitsUtils.create_columns_from_iterabletable(it2, :all)
+columns5, names5 = TableTraitsUtils.create_columns_from_iterabletable(it2, sel_cols=:all)
 @test columns5[1] == [1,2]
 @test columns5[2] == [1.,2.]
 @test names5 == [:a, :b]
 
-columns6, names6 = TableTraitsUtils.create_columns_from_iterabletable(it2, [2])
+columns6, names6 = TableTraitsUtils.create_columns_from_iterabletable(it2, sel_cols=[2])
 @test columns6[1] == [1.,2.]
 @test names6 == [:b]
 
-columns_with_nulls = (Nullable{Int}[Nullable(3), Nullable(2), Nullable{Int}()], [2.,5.,9.], Nullable{String}[Nullable("a"), Nullable{String}(), Nullable("b")])
+columns_with_nulls = (Union{Int,Missing}[3, 2, missing], Float64[2.,5.,9.], Union{String,Missing}["a", missing, "b"])
 it3 = TableTraitsUtils.create_tableiterator(columns_with_nulls, names)
 
 columns7, names7 = TableTraitsUtils.create_columns_from_iterabletable(it3)

test/test_source_without_length.jl

@@ -1,26 +1,18 @@
-using NamedTuples
-
 struct TestSourceWithoutLength
 end
 
 function Base.eltype(iter::TestSourceWithoutLength)
-    return @NT(a::Int, b::Float64)
+    return NamedTuple{(:a, :b), Tuple{Int, Float64}}
 end
 
-Base.iteratorsize(::Type{T}) where {T <: TestSourceWithoutLength} = Base.SizeUnknown()
-
-function Base.start(iter::TestSourceWithoutLength)
-    return 1
-end
+Base.IteratorSize(::Type{T}) where {T <: TestSourceWithoutLength} = Base.SizeUnknown()
 
-function Base.next(iter::TestSourceWithoutLength, state)
+function Base.iterate(iter::TestSourceWithoutLength, state=1)
     if state==1
-        return @NT(a=1, b=1.), 2
+        return (a=1, b=1.), 2
     elseif state==2
-        return @NT(a=2, b=2.), 3
+        return (a=2, b=2.), 3
+    else
+        return nothing
     end
 end
-
-function Base.done(iter::TestSourceWithoutLength, state)
-    return state>2
-end