Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Python: Small fixes #7989

Merged
merged 4 commits into from
Jul 25, 2023
Merged

Python: Small fixes #7989

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
145ba24
Python: Small improvements
Fokko Jul 5, 2023
13b60a5
Fix the test
Fokko Jul 5, 2023
76aa0c1
Python: Add some more missing defaults
Fokko Jul 6, 2023
6f99eda
Update tests as well
Fokko Jul 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions python/pyiceberg/table/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,53 +220,53 @@ class TableUpdate(IcebergBaseModel):


class UpgradeFormatVersionUpdate(TableUpdate):
action = TableUpdateAction.upgrade_format_version
action: TableUpdateAction = TableUpdateAction.upgrade_format_version
format_version: int = Field(alias="format-version")


class AddSchemaUpdate(TableUpdate):
action = TableUpdateAction.add_schema
action: TableUpdateAction = TableUpdateAction.add_schema
schema_: Schema = Field(alias="schema")


class SetCurrentSchemaUpdate(TableUpdate):
action = TableUpdateAction.set_current_schema
action: TableUpdateAction = TableUpdateAction.set_current_schema
schema_id: int = Field(
alias="schema-id", description="Schema ID to set as current, or -1 to set last added schema", default=-1
)


class AddPartitionSpecUpdate(TableUpdate):
action = TableUpdateAction.add_spec
action: TableUpdateAction = TableUpdateAction.add_spec
spec: PartitionSpec


class SetDefaultSpecUpdate(TableUpdate):
action = TableUpdateAction.set_default_spec
action: TableUpdateAction = TableUpdateAction.set_default_spec
spec_id: int = Field(
alias="spec-id", description="Partition spec ID to set as the default, or -1 to set last added spec", default=-1
)


class AddSortOrderUpdate(TableUpdate):
action = TableUpdateAction.add_sort_order
action: TableUpdateAction = TableUpdateAction.add_sort_order
sort_order: SortOrder = Field(alias="sort-order")


class SetDefaultSortOrderUpdate(TableUpdate):
action = TableUpdateAction.set_default_sort_order
action: TableUpdateAction = TableUpdateAction.set_default_sort_order
sort_order_id: int = Field(
alias="sort-order-id", description="Sort order ID to set as the default, or -1 to set last added sort order", default=-1
)


class AddSnapshotUpdate(TableUpdate):
action = TableUpdateAction.add_snapshot
action: TableUpdateAction = TableUpdateAction.add_snapshot
snapshot: Snapshot


class SetSnapshotRefUpdate(TableUpdate):
action = TableUpdateAction.set_snapshot_ref
action: TableUpdateAction = TableUpdateAction.set_snapshot_ref
ref_name: str = Field(alias="ref-name")
type: Literal["tag", "branch"]
snapshot_id: int = Field(alias="snapshot-id")
Expand All @@ -276,27 +276,27 @@ class SetSnapshotRefUpdate(TableUpdate):


class RemoveSnapshotsUpdate(TableUpdate):
action = TableUpdateAction.remove_snapshots
action: TableUpdateAction = TableUpdateAction.remove_snapshots
snapshot_ids: List[int] = Field(alias="snapshot-ids")


class RemoveSnapshotRefUpdate(TableUpdate):
action = TableUpdateAction.remove_snapshot_ref
action: TableUpdateAction = TableUpdateAction.remove_snapshot_ref
ref_name: str = Field(alias="ref-name")


class SetLocationUpdate(TableUpdate):
action = TableUpdateAction.set_location
action: TableUpdateAction = TableUpdateAction.set_location
location: str


class SetPropertiesUpdate(TableUpdate):
action = TableUpdateAction.set_properties
action: TableUpdateAction = TableUpdateAction.set_properties
updates: Dict[str, str]


class RemovePropertiesUpdate(TableUpdate):
action = TableUpdateAction.remove_properties
action: TableUpdateAction = TableUpdateAction.remove_properties
removals: List[str]


Expand Down
2 changes: 1 addition & 1 deletion python/pyiceberg/table/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ def construct_refs(cls, data: Dict[str, Any]) -> Dict[str, Any]:
default_spec_id: int = Field(alias="default-spec-id", default=INITIAL_SPEC_ID)
"""ID of the “current” spec that writers should use by default."""

last_partition_id: Optional[int] = Field(alias="last-partition-id")
last_partition_id: Optional[int] = Field(alias="last-partition-id", default=None)
"""An integer; the highest assigned partition field ID across all
partition specs for the table. This is used to ensure partition fields
are always assigned an unused ID when evolving specs."""
Expand Down
10 changes: 6 additions & 4 deletions python/pyiceberg/table/snapshots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,13 @@ def __repr__(self) -> str:

class Snapshot(IcebergBaseModel):
snapshot_id: int = Field(alias="snapshot-id")
parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id")
parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id", default=None)
sequence_number: Optional[int] = Field(alias="sequence-number", default=None)
timestamp_ms: int = Field(alias="timestamp-ms")
manifest_list: Optional[str] = Field(alias="manifest-list", description="Location of the snapshot's manifest list file")
summary: Optional[Summary] = Field()
manifest_list: Optional[str] = Field(
alias="manifest-list", description="Location of the snapshot's manifest list file", default=None
)
summary: Optional[Summary] = Field(default=None)
schema_id: Optional[int] = Field(alias="schema-id", default=None)

def __str__(self) -> str:
Expand All @@ -128,5 +130,5 @@ class MetadataLogEntry(IcebergBaseModel):


class SnapshotLogEntry(IcebergBaseModel):
snapshot_id: str = Field(alias="snapshot-id")
snapshot_id: int = Field(alias="snapshot-id")
timestamp_ms: int = Field(alias="timestamp-ms")
2 changes: 1 addition & 1 deletion python/tests/table/test_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def test_serialize_v1(example_table_metadata_v1: Dict[str, Any]) -> None:

def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]) -> None:
table_metadata = TableMetadataV2(**example_table_metadata_v2).json()
expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}"""
expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}"""
assert table_metadata == expected


Expand Down