Flexible vcat

src/abstractdataframe/abstractdataframe.jl

@@ -983,35 +983,88 @@ julia> vcat(df1, df2)
 │ 6   │ 6     │ 6     │
 ```
 """
-Base.vcat(df::AbstractDataFrame) = df
-Base.vcat(dfs::AbstractDataFrame...) = _vcat(collect(dfs))
-function _vcat(dfs::AbstractVector{<:AbstractDataFrame})
+Base.vcat(df::AbstractDataFrame; 
+        widen::Bool = false,  
+        keep::Union{Nothing, Vector{Symbol}} = nothing) = df
+
+Base.vcat(dfs::AbstractDataFrame...;
+          widen::Bool = false, 
+          missing = missing,
+          keep::Union{Nothing, Vector{Symbol}} = nothing) = 
+    _vcat(collect(dfs); widen = widen, keep = keep)
+
+function _vcat(dfs::AbstractVector{<:AbstractDataFrame}; 
+               widen::Bool = false, 
+               keep::Union{Nothing, Vector{Symbol}} = nothing)
+
     isempty(dfs) && return DataFrame()
-    allheaders = map(names, dfs)
-    uniqueheaders = unique(allheaders)
-    unionunique = union(uniqueheaders...)
-    intersectunique = intersect(uniqueheaders...)
-    coldiff = setdiff(unionunique, intersectunique)
-
-    if !isempty(coldiff)
+    # array of all headers
+    @show allheaders = map(names, dfs)
+    # unique arrays of all headers
+    @show uniqueheaders = unique(allheaders)
+    # Array of all the unique headers
+    @show unionunique = union(uniqueheaders...)
+    # Intersection of all unique headers 
+    @show intersectunique = intersect(uniqueheaders...)
+    # get the elements that are not present in everything
+    @show coldiff = setdiff(unionunique, intersectunique)
+
+    @show keep
+    if (widen == false) && !isempty(coldiff) && keep == nothing
         # if any DataFrames are a full superset of names, skip them
         filter!(u -> Set(u) != Set(unionunique), uniqueheaders)
         estrings = Vector{String}(undef, length(uniqueheaders))
         for (i, u) in enumerate(uniqueheaders)
-            matching = findall(h -> u == h, allheaders)
-            headerdiff = setdiff(coldiff, u)
-            cols = join(headerdiff, ", ", " and ")
-            args = join(matching, ", ", " and ")
+            @show matching = findall(h -> u == h, allheaders)
+            @show headerdiff = setdiff(coldiff, u)
+            @show cols = join(headerdiff, ", ", " and ")
+            @show args = join(matching, ", ", " and ")
             estrings[i] = "column(s) $cols are missing from argument(s) $args"
         end
         throw(ArgumentError(join(estrings, ", ", ", and ")))
     end
 
-    header = allheaders[1]
+    # TODO: make sure that `keep` throws a good error if 
+    # it a) isn't in `allheaders` or b) isn't a subset of `unionunique`
+    header = (keep == nothing) ? unionunique : keep 
+
+    if keep == nothing 
+        # Make the order of the names match the order of the dataframes inputted
+        header = let unionunique = unionunique, allheaders = allheaders
+            t = [filter(h -> h in unionunique, head) for head in allheaders]
+            reduce((a, b) -> [a; setdiff(b, a)], t)
+        end
+    else 
+        header = keep 
+    end
+
     length(header) == 0 && return DataFrame()
     cols = Vector{AbstractVector}(undef, length(header))
     for (i, name) in enumerate(header)
-        data = [df[name] for df in dfs]
+        # TODO: replace with commented out code after getindex deprecation
+        # data = [df[name] for df in dfs]
+        # the code below assumes that only DataFrame and SubDataFrame
+        # are subtypes of AbstractDataFrame
+        # it should be removed ASAP after deprecation
+        data = map(dfs) do df
+            if df isa DataFrame 
+                if haskey(df, name)
+                    return df[name] 
+                else
+                    # TODO: make this more efficient by not creating a 
+                    # full array of missing values. Instead, implement
+                    # this in the copyto! stage
+                    return fill(missing, nrow(df))
+                end
+            else 
+                if haskey(df, name)
+                    return view(parent(df)[name], rows(df))
+                else 
+                    return fill(missing, length(rows(df)))
+                end
+            end
+        end
+
         lens = map(length, data)
         T = mapreduce(eltype, promote_type, data)
         cols[i] = Tables.allocatecolumn(T, sum(lens))