Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Core, API, Spec: Metadata Row Lineage #11948

Merged
merged 22 commits into from
Feb 1, 2025
Merged

Core, API, Spec: Metadata Row Lineage #11948

Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
715dbba
Core: Allow setting Row Lineage and Last-Row-ID in TableMetadata
RussellSpitzer Jan 9, 2025
4039869
API, CORE, SPEC: Adds Snapshot Fields for Row Lineage
RussellSpitzer Jan 10, 2025
c416acf
Cleanup Unused Test Variables
RussellSpitzer Jan 10, 2025
1b4f0f6
Core: Add tests for Append Pathways
RussellSpitzer Jan 10, 2025
75c1ad4
Review Comments
RussellSpitzer Jan 13, 2025
3ed2844
Adds some additional testing
RussellSpitzer Jan 13, 2025
a58f7b4
Fix Checkstyle
RussellSpitzer Jan 13, 2025
5811521
Reorder Imports
RussellSpitzer Jan 13, 2025
f472497
Add ability to set row-lineage via table property
RussellSpitzer Jan 13, 2025
1023d81
Spotless Apply
RussellSpitzer Jan 14, 2025
d7c5416
Review Comments
RussellSpitzer Jan 15, 2025
2c0546d
Test Fix
RussellSpitzer Jan 15, 2025
f048cc3
Reviewer Comments
RussellSpitzer Jan 17, 2025
a929c32
Spotless
RussellSpitzer Jan 21, 2025
b51efa8
Use Test Template
RussellSpitzer Jan 21, 2025
c567421
Spotless
RussellSpitzer Jan 21, 2025
d2e016f
Switch Test Parameters to Class Parameter
RussellSpitzer Jan 22, 2025
9b574ff
Missing Line in last Commit
RussellSpitzer Jan 23, 2025
8d143c9
Reformat error messages
RussellSpitzer Jan 23, 2025
e789fda
Update Test Error Message Checking
RussellSpitzer Jan 23, 2025
6fac1a4
Remove overly paranoid asserts
RussellSpitzer Jan 24, 2025
61f18c2
Reorder asserts in getBoolOrNull Test
RussellSpitzer Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Switch Test Parameters to Class Parameter
  • Loading branch information
@RussellSpitzer
RussellSpitzer committed Jan 22, 2025
commit d2e016f09b265ea4aed56d02c651eb0b861b7098
27 changes: 15 additions & 12 deletions core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ private static List<Integer> formatVersion() {
return Ints.asList(TestHelpers.ALL_VERSIONS);
}

@Parameter
private int formatVersion;

private static final String TEST_LOCATION = "s3://bucket/test/location";

private static final Schema TEST_SCHEMA =
RussellSpitzer marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -50,7 +53,7 @@ private static List<Integer> formatVersion() {
Types.NestedField.required(2, "y", Types.LongType.get(), "comment"),
Types.NestedField.required(3, "z", Types.LongType.get()));

private TableMetadata baseMetadata(int formatVersion) {
private TableMetadata baseMetadata() {
return TableMetadata.buildFromEmpty(formatVersion)
.enableRowLineage()
.addSchema(TEST_SCHEMA)
Expand All @@ -68,7 +71,7 @@ public void cleanup() {
}

@TestTemplate
public void testRowLineageSupported(int formatVersion) {
public void testRowLineageSupported() {
if (formatVersion >= TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE) {
assertThat(TableMetadata.buildFromEmpty(formatVersion).enableRowLineage()).isNotNull();
} else {
Expand All @@ -79,12 +82,12 @@ public void testRowLineageSupported(int formatVersion) {
}

@TestTemplate
public void testSnapshotAddition(int formatVersion) {
public void testSnapshotAddition() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

long newRows = 30L;

TableMetadata base = baseMetadata(formatVersion);
TableMetadata base = baseMetadata();

Snapshot addRows =
new BaseSnapshot(
Expand All @@ -105,12 +108,12 @@ public void testSnapshotAddition(int formatVersion) {
}

@TestTemplate
public void testInvalidSnapshotAddition(int formatVersion) {
public void testInvalidSnapshotAddition() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

Long newRows = 30L;

TableMetadata base = baseMetadata(formatVersion);
TableMetadata base = baseMetadata();

Snapshot invalidLastRow =
new BaseSnapshot(
Expand All @@ -131,7 +134,7 @@ public void testInvalidSnapshotAddition(int formatVersion) {
}

@TestTemplate
public void testFastAppend(int formatVersion) {
public void testFastAppend() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

TestTables.TestTable table =
Expand All @@ -157,7 +160,7 @@ public void testFastAppend(int formatVersion) {
}

@TestTemplate
public void testAppend(int formatVersion) {
public void testAppend() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

TestTables.TestTable table =
Expand All @@ -183,7 +186,7 @@ public void testAppend(int formatVersion) {
}

@TestTemplate
public void testAppendBranch(int formatVersion) {
public void testAppendBranch() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);
// Appends to a branch should still change last-row-id even if not on main, these changes
// should also affect commits to main
Expand Down Expand Up @@ -222,7 +225,7 @@ public void testAppendBranch(int formatVersion) {
}

@TestTemplate
public void testDeletes(int formatVersion) {
public void testDeletes() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

TestTables.TestTable table =
Expand Down Expand Up @@ -254,7 +257,7 @@ public void testDeletes(int formatVersion) {
}

@TestTemplate
public void testReplace(int formatVersion) {
public void testReplace() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

TestTables.TestTable table =
Expand Down Expand Up @@ -286,7 +289,7 @@ public void testReplace(int formatVersion) {
}

@TestTemplate
public void testEnableRowLineageViaProperty(int formatVersion) {
public void testEnableRowLineageViaProperty() {
assumeThat(formatVersion).isGreaterThanOrEqualTo(TableMetadata.MIN_FORMAT_VERSION_ROW_LINEAGE);

TestTables.TestTable table =
Expand Down
Loading