Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow any data structure that implements Table.Reader #75

Merged
merged 2 commits into from
Apr 30, 2023
Merged

Allow any data structure that implements Table.Reader #75

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ed3eff
Allow any data structure that implements Table.Reader
cristineguadelupe Apr 28, 2023
43e5b17
Update the first line of the pipeline
cristineguadelupe Apr 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/assets/data_transform_cell/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ export async function init(ctx, payload) {
<div class="root">
<BaseSelect
name="data_frame"
label="Data Frame"
label="Data"
v-model="rootFields.data_frame"
:options="dataFrameVariables"
:required
Expand Down
38 changes: 35 additions & 3 deletions lib/kino_explorer/data_transform_cell.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ defmodule KinoExplorer.DataTransformCell do
def handle_info({:scan_binding_result, binding, data_frame_alias, missing_require}, ctx) do
data_frames =
for {key, val} <- binding,
is_struct(val, DataFrame),
valid_data(val),
do: %{
variable: Atom.to_string(key),
data: val
Expand Down Expand Up @@ -386,6 +386,7 @@ defmodule KinoExplorer.DataTransformCell do
|> Map.put("operations", ctx.assigns.operations)
|> Map.put("data_frame_alias", ctx.assigns.data_frame_alias)
|> Map.put("missing_require", ctx.assigns.missing_require)
|> Map.put("is_data_frame", is_data_frame?(ctx))
end

@impl true
Expand All @@ -410,6 +411,8 @@ defmodule KinoExplorer.DataTransformCell do
|> Enum.chunk_by(& &1.operation_type)
|> Enum.map(&(to_quoted(&1) |> Map.merge(%{module: attrs.data_frame_alias})))

nodes = if attrs.is_data_frame, do: nodes, else: [build_df() | nodes]

root = build_root(df)

nodes
Expand Down Expand Up @@ -487,6 +490,10 @@ defmodule KinoExplorer.DataTransformCell do
end
end

defp build_df() do
%{args: [], field: :new, module: Explorer.DataFrame, name: :new}
end

defp build_var(acc, nil), do: acc

defp build_var(acc, var) do
Expand Down Expand Up @@ -725,10 +732,14 @@ defmodule KinoExplorer.DataTransformCell do

defp update_data_options([operation], ctx, data_frame) do
data_frames = ctx.assigns.data_frames
df = Enum.find_value(data_frames, &(&1.variable == data_frame && Map.get(&1, :data)))

data_options =
if df = Enum.find_value(data_frames, &(&1.variable == data_frame && Map.get(&1, :data))),
do: DataFrame.dtypes(df)
case df do
nil -> nil
%DataFrame{} -> DataFrame.dtypes(df)
_ -> df |> DataFrame.new() |> DataFrame.dtypes()
end

[Map.put(operation, "data_options", data_options)]
end
Expand Down Expand Up @@ -777,6 +788,7 @@ defmodule KinoExplorer.DataTransformCell do
|> Map.put("operations", partial_operations)
|> Map.put("data_frame_alias", Explorer.DataFrame)
|> Map.put("missing_require", Explorer.DataFrame)
|> Map.put("is_data_frame", is_data_frame?(ctx))
end

defp maybe_update_datalist(%{"operation_type" => "filters"} = operation, df) do
Expand All @@ -789,4 +801,24 @@ defmodule KinoExplorer.DataTransformCell do
end

defp maybe_update_datalist(operation, _df), do: operation

defp valid_data(%DataFrame{}), do: true

defp valid_data(data) do
with true <- implements?(Table.Reader, data),
{_, %{columns: [_ | _] = columns}, _} <- Table.Reader.init(data),
true <- Enum.all?(columns, &implements?(String.Chars, &1)) do
true
else
_ -> false
end
end

defp implements?(protocol, value), do: protocol.impl_for(value) != nil

defp is_data_frame?(ctx) do
df = ctx.assigns.root_fields["data_frame"]
data = Enum.find_value(ctx.assigns.data_frames, &(&1.variable == df && Map.get(&1, :data)))
is_struct(data, DataFrame)
end
end
68 changes: 62 additions & 6 deletions test/kino_explorer/data_transform_cell_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ defmodule KinoExplorer.DataTransformCellTest do
"data_frame" => "people",
"assign_to" => nil,
"data_frame_alias" => Explorer.DataFrame,
"missing_require" => nil
"missing_require" => nil,
"is_data_frame" => true
}

@base_operations %{
Expand Down Expand Up @@ -69,17 +70,23 @@ defmodule KinoExplorer.DataTransformCellTest do
assert source == ""
end

test "finds Explorer DataFrames in binding and sends the data options to the client" do
test "finds valid data in binding and sends the data options to the client" do
{kino, _source} = start_smart_cell!(DataTransformCell, %{})

teams = teams_df()
people = people_df()

simple_data = [
%{id: 1, name: "Elixir", website: "https://elixir-lang.org"},
%{id: 2, name: "Erlang", website: "https://www.erlang.org"}
]

invalid_data = %{self() => [1, 2], :y => [1, 2]}

env = Code.env_for_eval([])
DataTransformCell.scan_binding(kino.pid, binding(), env)

data_frame_variables = ["people", "teams"]
data_frame_variables = ["people", "simple_data", "teams"]

assert_broadcast_event(kino, "set_available_data", %{
"data_frame_variables" => ^data_frame_variables,
Expand All @@ -101,6 +108,43 @@ defmodule KinoExplorer.DataTransformCellTest do
})
end

test "initial data options for a non data_frame data" do
{kino, _source} = start_smart_cell!(DataTransformCell, %{})

simple_data = [
%{id: 1, name: "Elixir", website: "https://elixir-lang.org"},
%{id: 2, name: "Erlang", website: "https://www.erlang.org"}
]

env = Code.env_for_eval([])
DataTransformCell.scan_binding(kino.pid, binding(), env)

data_frame_variables = ["simple_data"]

assert_broadcast_event(kino, "set_available_data", %{
"data_frame_variables" => ^data_frame_variables,
"fields" => %{
operations: [
%{
"active" => true,
"column" => nil,
"data_options" => %{
"id" => :integer,
"name" => :string,
"website" => :string
},
"datalist" => [],
"filter" => nil,
"operation_type" => "filters",
"type" => "string",
"value" => nil
}
],
root_fields: %{"assign_to" => nil, "data_frame" => "simple_data"}
}
})
end

describe "code generation" do
test "source for a data frame without operations" do
attrs = build_attrs(%{})
Expand All @@ -110,6 +154,15 @@ defmodule KinoExplorer.DataTransformCellTest do
"""
end

test "source for a data without operations" do
root = %{"data_frame" => "simple_data", "is_data_frame" => false}
attrs = build_attrs(root, %{})

assert DataTransformCell.to_source(attrs) == """
simple_data |> Explorer.DataFrame.new()\
"""
end

test "source for a data frame with sorting" do
attrs =
build_attrs(%{
Expand Down Expand Up @@ -855,7 +908,8 @@ defmodule KinoExplorer.DataTransformCellTest do
"data_frame" => "people",
"assign_to" => "exported_df",
"data_frame_alias" => DF,
"missing_require" => nil
"missing_require" => nil,
"is_data_frame" => true
}

operations = [
Expand Down Expand Up @@ -1403,7 +1457,8 @@ defmodule KinoExplorer.DataTransformCellTest do
connect(kino)
teams = teams_df()
env = Code.env_for_eval([])
DataTransformCell.scan_binding(kino.pid, binding(), env)
binding = binding() |> Keyword.delete(:operations)
DataTransformCell.scan_binding(kino.pid, binding, env)

updated_operations =
Enum.map(
Expand Down Expand Up @@ -1455,7 +1510,8 @@ defmodule KinoExplorer.DataTransformCellTest do
connect(kino)
teams = teams_df()
env = Code.env_for_eval([])
DataTransformCell.scan_binding(kino.pid, binding(), env)
binding = binding() |> Keyword.delete(:operations)
DataTransformCell.scan_binding(kino.pid, binding, env)

{operations, [sort]} = Enum.split(operations, 3)

Expand Down