Skip to content

Commit

Permalink
feat: working joins
Browse files Browse the repository at this point in the history
  • Loading branch information
@freedmand
freedmand committed Nov 1, 2022
1 parent fd792f4 commit 2db496b
Show file tree
Hide file tree
Showing 5 changed files with 214 additions and 71 deletions.
132 changes: 91 additions & 41 deletions src/components/Matcher.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { useEffect } from "react";
import { AppReducer, Match, MatchRow, ProcessingState } from "../state";
import { newEditDistance } from "../utils/match";
import { filterByValue, joinNorm, uniq } from "../utils/uniq";

export function Progress({ progress }: { progress: number }) {
const numBars = 40;
Expand Down Expand Up @@ -47,50 +48,99 @@ export function Matcher({
}
}) as [(string | undefined)[], (string | undefined)[]];

const results: MatchRow[] = [];
for (let i = 0; i < leftValues.length; i++) {
timeouts.push(
setTimeout(() => {
// Rank everything
const matches: Match[] = [];
const row = results.length;
for (let j = 0; j < rightValues.length; j++) {
const score = newEditDistance(leftValues[i], rightValues[j]);
matches.push({
score,
value: rightValues[j],
meta: rightMetas[j],
index: j,
col: 0,
const [leftJoins, rightJoins] = (["left", "right"] as const).map((side) => {
const join = app.columnSelections.join;
return join
? app.tables[join[side].tableIndex].rows.map((row) =>
joinNorm(row[join[side].column])
)
: null;
});
const hasJoin = leftJoins != null && rightJoins != null;
const uniqueJoinValues = uniq([
...(leftJoins || []),
...(rightJoins || []),
]);

const joins: [string, string[], string[]][] = hasJoin
? uniqueJoinValues
.map<[string, string[], string[]]>((joiner) => [
joiner,
filterByValue(leftJoins, joiner, leftValues),
filterByValue(rightJoins, joiner, rightValues),
])
.filter(
(x: [string, string[], string[]]) =>
x[1].length > 0 && x[2].length > 0
)
: // Default join of just left and right values
[["default", leftValues, rightValues]];
console.log(joins);

const overallResults: [string, MatchRow[]][] = [];

for (let joinIndex = 0; joinIndex < joins.length; joinIndex++) {
console.log("JOIN INDEXX", joinIndex);
const [joiner, leftValues, rightValues] = joins[joinIndex];
const results: MatchRow[] = [];
for (let i = 0; i < leftValues.length; i++) {
timeouts.push(
setTimeout(() => {
// Rank everything
const matches: Match[] = [];
const row = results.length;
for (let j = 0; j < rightValues.length; j++) {
const score = newEditDistance(leftValues[i], rightValues[j]);
matches.push({
score,
value: rightValues[j],
meta: rightMetas[j],
index: j,
col: 0,
row,
});
}
matches.sort((a, b) => a.score - b.score);
for (let z = 0; z < matches.length; z++) {
// Assign match cols
matches[z].col = z;
}
results.push({
value: leftValues[i],
meta: leftMetas[i],
index: i,
rankedMatches: matches,
row,
});
}
matches.sort((a, b) => a.score - b.score);
for (let z = 0; z < matches.length; z++) {
// Assign match cols
matches[z].col = z;
}
results.push({
value: leftValues[i],
meta: leftMetas[i],
index: i,
rankedMatches: matches,
row,
});
// Update progress
if (i === leftValues.length - 1) {
reducer({
type: "FinishProcessing",
results: results,
});
} else {
reducer({
type: "UpdateProgress",
progress: (i + 1) / leftValues.length,

// Finalize results
if (i === leftValues.length - 1) {
overallResults.push([joiner, results]);
}

// Update progress
console.log({
i,
joinIndex,
iL: leftValues.length,
jL: joins.length,
});
}
}, 16)
);
if (i === leftValues.length - 1 && joinIndex === joins.length - 1) {
console.log("DONE", overallResults);
reducer({
type: "FinishProcessing",
results: overallResults,
});
} else {
reducer({
type: "UpdateProgress",
progress:
(joinIndex + (i + 1) / leftValues.length) / joins.length,
});
}
}, 16)
);
}
}

return () => {
Expand Down
79 changes: 58 additions & 21 deletions src/components/MatchingTable.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,12 @@ export function MatchingTable({
);
const [showMeta, setShowMeta] = useState<"show" | "hide">("show");

if (app.matches.length === 0) return null;
const allJoins = Object.keys(app.matches);
const [join, setJoin] = useState(allJoins[0] || "");

const numColumns = app.matches[0].rankedMatches.length + 1;
if (allJoins.length === 0) return null;

const numColumns = app.matches[join][0].rankedMatches.length + 1;
const columns: GridColumn[] = [
{
title: `SOURCE — ${app.columnSelections.left.column} (${app.columnSelections.left.tableName})`,
Expand Down Expand Up @@ -71,18 +74,18 @@ export function MatchingTable({
// }

function getUserMatched(matchCell: Match): boolean {
return app.userMatches[`${matchCell.col},${matchCell.row}`];
return app.userMatches[join][`${matchCell.col},${matchCell.row}`];
}

function getAllUserMatches(): Match[] {
const entries = Object.entries(app.userMatches);
const entries = Object.entries(app.userMatches[join]);
const results: Match[] = [];
for (const [key, value] of entries) {
if (!value) continue;
const parts = key.split(",");
const x = parseInt(parts[0]);
const y = parseInt(parts[1]);
results.push(app.matches[y].rankedMatches[x]);
results.push(app.matches[join][y].rankedMatches[x]);
}
return results;
}
Expand All @@ -94,17 +97,24 @@ export function MatchingTable({
return userMatchTexts.includes(match.value);
}

const matchEntries = Object.entries(app.userMatches);
const rowsWithMatches: { [row: number]: boolean } = {};
for (const [key, value] of matchEntries) {
if (value) {
const row = parseInt(key.split(",")[1]);
rowsWithMatches[row] = true;
const matchEntries: [string, [string, boolean][]][] = Object.entries(
app.userMatches
).map(([k, v]) => [k, Object.entries(v)]);
const rowsWithMatches: { [join: string]: { [row: number]: boolean } } = {};
for (const [join, subEntries] of matchEntries) {
for (const [key, value] of subEntries) {
if (value) {
const row = parseInt(key.split(",")[1]);
if (rowsWithMatches[join] == null) {
rowsWithMatches[join] = {};
}
rowsWithMatches[join][row] = true;
}
}
}

function rowHasMatch(row: number): boolean {
return rowsWithMatches[row];
function rowHasMatch(row: number, subJoin = join): boolean {
return rowsWithMatches[subJoin]?.[row];
}

function drawTextWithMatches(
Expand Down Expand Up @@ -170,7 +180,7 @@ export function MatchingTable({
}

const filteredMatchRows = new FilteredMatchRows(
app.matches,
app.matches[join],
(cell) =>
colFilter === "all"
? true
Expand Down Expand Up @@ -244,12 +254,13 @@ export function MatchingTable({
gridSelection.current.range,
...gridSelection.current.rangeStack,
],
join,
});
if (
gridSelection.current.range.width === 1 &&
gridSelection.current.range.height === 1 &&
gridSelection.current.rangeStack.length === 0 &&
gridSelection.current.range.y < app.matches.length - 1
gridSelection.current.range.y < filteredMatchRows.numRows - 1
) {
// Advance to the next row
setGridSelection({
Expand Down Expand Up @@ -290,6 +301,7 @@ export function MatchingTable({
...gridSelection.current.rangeStack,
],
data: filteredMatchRows,
join,
forceState: false,
});
}
Expand Down Expand Up @@ -361,7 +373,7 @@ export function MatchingTable({
text,
col === 1 || isUserMatch
? ""
: app.matches[matchCell.row].value,
: app.matches[join][matchCell.row].value,
rect.x + 5,
rect.y + rect.height / 2 + 1,
sizeOffset
Expand All @@ -371,7 +383,9 @@ export function MatchingTable({
ctx,
"fill",
text,
col === 1 || isUserMatch ? "" : app.matches[matchCell.row].value,
col === 1 || isUserMatch
? ""
: app.matches[join][matchCell.row].value,
rect.x + 5,
rect.y + rect.height / 2 + 1,
sizeOffset
Expand All @@ -390,7 +404,18 @@ export function MatchingTable({
}}
getCellContent={(cell) => {
const [column, row] = cell;
const filteredMatches = filteredMatchRows.getRow(row);
let filteredMatches;
try {
filteredMatches = filteredMatchRows.getRow(row);
} catch (e) {
// This happens when the render is called while data changes
return {
kind: GridCellKind.Text,
data: "",
allowOverlay: false,
displayData: "",
};
}

if (column === 0) {
return {
Expand Down Expand Up @@ -453,6 +478,18 @@ export function MatchingTable({

<ResetButton slim={true} app={app} reducer={reducer} />

<select onInput={(e) => setJoin((e.target as HTMLSelectElement).value)}>
{allJoins.map((join) => (
<option key={join} value={join}>
{join} (
{app.matches[join]
.filter((matchRow: MatchRow) => rowHasMatch(matchRow.row, join))
.length.toLocaleString()}{" "}
/ {app.matches[join].length.toLocaleString()})
</option>
))}
</select>

<select
onInput={(e) =>
setRowFilter(
Expand All @@ -464,18 +501,18 @@ export function MatchingTable({
}
>
<option value="all">
Show all rows ({app.matches.length.toLocaleString()})
Show all rows ({app.matches[join].length.toLocaleString()})
</option>
<option value="incomplete">
Show incomplete rows (
{app.matches
{app.matches[join]
.filter((matchRow: MatchRow) => !rowHasMatch(matchRow.row))
.length.toLocaleString()}
)
</option>
<option value="complete">
Show complete rows (
{app.matches
{app.matches[join]
.filter((matchRow: MatchRow) => rowHasMatch(matchRow.row))
.length.toLocaleString()}
)
Expand Down
Loading

0 comments on commit 2db496b

Please sign in to comment.