added TableDataset

Project.toml

@@ -6,10 +6,12 @@ version = "0.1.0"
 [deps]
 Animations = "27a7e980-b3e6-11e9-2bcd-0b925532e340"
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 DLPipelines = "e6530d7c-7faa-4ede-a0d6-9eff9baad396"
 DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e"
 DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataLoaders = "2e981812-ef13-4a9c-bfa0-ab13047b12a9"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"

src/datasets/Datasets.jl

@@ -126,6 +126,9 @@ using MLDataPattern: splitobs
 import LearnBase
 using Colors
 using FixedPointNumbers
+using DataFrames
+using Tables
+using CSV
 
 include("fastaidatasets.jl")
 

src/datasets/containers.jl

@@ -1,5 +1,3 @@
-
-
 # FileDataset
 
 struct FileDataset
@@ -45,5 +43,41 @@ isimagefile(file::File) = isimagefile(file.name)
 isimagefile(file::String) = occursin(IMAGEFILE_REGEX, lowercase(file))
 const IMAGEFILE_REGEX = r"\.(gif|jpe?g|tiff?|png|webp|bmp)$"
 
+#TableDataset
+
+struct TableDataset{T}
+    table::T #Should implement Tables.jl interface
+    TableDataset{T}(table::T) where T = Tables.istable(table) ? new{T}(table) : error("Object doesn't implement Tables.jl interface")
+end
+
+TableDataset(table::T) where {T} = TableDataset{T}(table)
+TableDataset(path::AbstractPath) = TableDataset(DataFrame(CSV.File(path)))
+
+function LearnBase.getobs(dataset::FastAI.Datasets.TableDataset, idx)
+    if Tables.rowaccess(dataset.table)
+        row, _ = Iterators.peel(Iterators.drop(Tables.rows(dataset.table), idx - 1))
+        return row
+    elseif Tables.columnaccess(dataset.table)
+        colnames = Tables.columnnames(dataset.table)
+        rowvals = [Tables.getcolumn(dataset.table, i)[idx] for i in 1:length(colnames)]
+        return (; zip(colnames, rowvals)...)
+    else 
+        error("The Tables.jl implementation used should have either rowaccess or columnaccess.")
+    end
+end
+
+function LearnBase.nobs(dataset::TableDataset)
+    if Tables.columnaccess(dataset.table)
+        return length(Tables.getcolumn(dataset.table, 1))
+    elseif Tables.rowaccess(dataset.table)
+        return length(Tables.rows(dataset.table)) # length might not be defined, but has to be for this to work.
+    else 
+        error("The Tables.jl implementation used should have either rowaccess or columnaccess.")
+    end
+end
+
+LearnBase.getobs(dataset::TableDataset{<:DataFrame}, idx) = dataset.table[idx, :]
+LearnBase.nobs(dataset::TableDataset{<:DataFrame}) = nrow(dataset.table)
 
-## TODO: TableDataset
+LearnBase.getobs(dataset::TableDataset{<:CSV.File}, idx) = dataset.table[idx]
+LearnBase.nobs(dataset::TableDataset{<:CSV.File}) = length(dataset.table)

test/Project.toml

@@ -1,9 +1,15 @@
 [deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 DLPipelines = "e6530d7c-7faa-4ede-a0d6-9eff9baad396"
 DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
+ShowCases = "605ecd9f-84a6-4c9e-81e2-4798472b76a3"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestSetExtensions = "98d24dd4-01ad-11ea-1b02-c9a08f80db04"

test/datasets/containers.jl

@@ -0,0 +1,61 @@
+include("../imports.jl")
+
+@testset ExtendedTestSet "TableDataset" begin
+
+    @testset ExtendedTestSet "TableDataset from rowaccess table" begin
+        Tables.columnaccess(::Type{<:Tables.MatrixTable}) = false
+        Tables.rowaccess(::Type{<:Tables.MatrixTable}) = true
+
+        testtable = Tables.table([1 4.0 "7"; 2 5.0 "8"; 3 6.0 "9"])
+        td = TableDataset(testtable)
+
+        @test all(getobs(td, 1) .== [1, 4.0, "7"])
+        @test nobs(td) == 3
+    end
+
+    @testset ExtendedTestSet "TableDataset from columnaccess table" begin
+        Tables.columnaccess(::Type{<:Tables.MatrixTable}) = true
+        Tables.rowaccess(::Type{<:Tables.MatrixTable}) = false
+
+        testtable = Tables.table([1 4.0 "7"; 2 5.0 "8"; 3 6.0 "9"])
+        td = TableDataset(testtable)
+
+        @test [data for data in getobs(td, 2)] == [2, 5.0, "8"]
+        @test nobs(td) == 3
+
+        @test getobs(td, 1) isa NamedTuple
+    end
+
+    @testset ExtendedTestSet "TableDataset from DataFrames" begin
+        testtable = DataFrame(
+            col1=[1, 2, 3, 4, 5], 
+            col2=["a", "b", "c", "d", "e"], 
+            col3=[10, 20, 30, 40, 50], 
+            col4=["A", "B", "C", "D", "E"],
+            col5=[100., 200., 300., 400., 500.],
+            split=["train", "train", "train", "valid", "valid"]
+        )
+        td = TableDataset(testtable)
+        @test td isa TableDataset{<:DataFrame}
+
+        @test [data for data in getobs(td, 1)] == [1, "a", 10, "A", 100., "train"]
+        @test nobs(td) == 5 
+    end
+
+    @testset ExtendedTestSet "TableDataset from CSV" begin
+        open("test.csv", "w") do io
+            write(io, "col1,col2,col3,col4,col5, split\n1,a,10,A,100.,train")
+        end
+        testtable = CSV.File("test.csv")
+        td = TableDataset(testtable)
+        @test td isa TableDataset{<:CSV.File}
+        @test [data for data in getobs(td, 1)] == [1,
+                                                   "a",
+                                                   10,
+                                                   "A",
+                                                   100.,
+                                                   "train"]
+        @test nobs(td) == 1
+        rm("test.csv")
+    end
+end

test/imports.jl

@@ -12,6 +12,9 @@ using Flux.Optimise: Optimiser, apply!
 using StaticArrays
 using Test
 using TestSetExtensions
+using DataFrames
+using Tables
+using CSV
 
 ENV["DATADEPS_ALWAYS_ACCEPT"] = "true"
 include("testdata.jl")