Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: prefer usage of metadata and protocol fields #1935

Merged
merged 6 commits into from
Dec 4, 2023
Merged

refactor: prefer usage of metadata and protocol fields #1935

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7ce28ba
refactor: harmonize usage of table protocol and metadata
roeap Nov 18, 2023
97aa791
feat: add clusterinProvider field to add actions
roeap Nov 26, 2023
f32a06a
chore: remove deprecated function invocations
roeap Dec 3, 2023
1f670b6
fix: clusteringProvider data type
roeap Dec 3, 2023
2a3e023
chore: deprecate more fields on DeltaTable
roeap Dec 3, 2023
e730b78
Merge branch 'main' into meta-usage
roeap Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: deprecate more fields on DeltaTable
  • Loading branch information
@roeap
roeap committed Dec 3, 2023
commit 2a3e023ef989043774281b02fd681d4cbfae4981
24 changes: 13 additions & 11 deletions crates/deltalake-core/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ pub fn crate_version() -> &'static str {

#[cfg(test)]
mod tests {
use itertools::Itertools;

use super::*;
use crate::table::PeekCommit;
use std::collections::HashMap;
Expand All @@ -207,7 +209,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet"),
Path::from("part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet"),
Expand Down Expand Up @@ -241,8 +243,8 @@ mod tests {
table_to_update.update().await.unwrap();

assert_eq!(
table_newest_version.get_files(),
table_to_update.get_files()
table_newest_version.get_files_iter().collect_vec(),
table_to_update.get_files_iter().collect_vec()
);
}
#[tokio::test]
Expand All @@ -254,7 +256,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-b44fcdb0-8b06-4f3a-8606-f8311a96f6dc-c000.snappy.parquet"),
Path::from("part-00001-185eca06-e017-4dea-ae49-fc48b973e37e-c000.snappy.parquet"),
Expand All @@ -268,7 +270,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet"),
Path::from("part-00001-c373a5bd-85f0-4758-815e-7eb62007a15c-c000.snappy.parquet"),
Expand All @@ -282,7 +284,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-cb6b150b-30b8-4662-ad28-ff32ddab96d2-c000.snappy.parquet"),
Path::from("part-00000-7c2deba3-1994-4fb8-bc07-d46c948aa415-c000.snappy.parquet"),
Expand All @@ -298,7 +300,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a1-c000.snappy.parquet"),
Path::from("part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe1-c000.snappy.parquet")
Expand Down Expand Up @@ -344,7 +346,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a1-c000.snappy.parquet"),
Path::from("part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe1-c000.snappy.parquet"),
Expand All @@ -355,7 +357,7 @@ mod tests {
assert_eq!(table.protocol().min_writer_version, 2);
assert_eq!(table.protocol().min_reader_version, 1);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a1-c000.snappy.parquet"),
Path::from("part-00001-911a94a2-43f6-4acb-8620-5e68c2654989-c000.snappy.parquet"),
Expand Down Expand Up @@ -483,7 +485,7 @@ mod tests {
.unwrap();

assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::parse(
"x=A%2FA/part-00007-b350e235-2832-45df-9918-6cab4f7578f7.c000.snappy.parquet"
Expand Down Expand Up @@ -683,7 +685,7 @@ mod tests {
.unwrap();
assert_eq!(table.version(), 2);
assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![Path::from(
"part-00000-7444aec4-710a-4a4c-8abe-3323499043e9.c000.snappy.parquet"
),]
Expand Down
3 changes: 2 additions & 1 deletion crates/deltalake-core/src/operations/convert_to_delta.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,7 @@ mod tests {
storage::config::StorageOptions,
Path,
};
use itertools::Itertools;
use pretty_assertions::assert_eq;
use std::fs;
use tempfile::tempdir;
Expand Down Expand Up @@ -501,7 +502,7 @@ mod tests {
"Testing location: {test_data_from:?}"
);

let mut files = table.get_files();
let mut files = table.get_files_iter().collect_vec();
files.sort();
assert_eq!(
files, expected_paths,
Expand Down
9 changes: 5 additions & 4 deletions crates/deltalake-core/src/table/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,7 @@ pub fn ensure_table_uri(table_uri: impl AsRef<str>) -> DeltaResult<Url> {
#[cfg(test)]
mod tests {
use super::*;
use itertools::Itertools;
use object_store::path::Path;

#[test]
Expand Down Expand Up @@ -607,7 +608,7 @@ mod tests {
);

assert_eq!(
table.get_files(),
table.get_files_iter().collect_vec(),
vec![
Path::from("part-00000-c9b90f86-73e6-46c8-93ba-ff6bfaf892a1-c000.snappy.parquet"),
Path::from("part-00000-04ec9591-0b73-459e-8d18-ba5711d6cbe1-c000.snappy.parquet")
Expand All @@ -623,7 +624,7 @@ mod tests {
.await
.unwrap();

assert!(table.get_files().is_empty(), "files should be empty");
assert_eq!(table.get_files_iter().count(), 0, "files should be empty");
assert!(
table.get_tombstones().next().is_none(),
"tombstones should be empty"
Expand All @@ -640,15 +641,15 @@ mod tests {
.unwrap();

assert_eq!(table.version(), 0);
assert!(table.get_files().is_empty(), "files should be empty");
assert_eq!(table.get_files_iter().count(), 0, "files should be empty");
assert!(
table.get_tombstones().next().is_none(),
"tombstones should be empty"
);

table.update().await.unwrap();
assert_eq!(table.version(), 1);
assert!(table.get_files().is_empty(), "files should be empty");
assert_eq!(table.get_files_iter().count(), 0, "files should be empty");
assert!(
table.get_tombstones().next().is_none(),
"tombstones should be empty"
Expand Down
6 changes: 6 additions & 0 deletions crates/deltalake-core/src/table/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -728,12 +728,14 @@ impl DeltaTable {
}

/// Returns a collection of file names present in the loaded state
#[deprecated(since = "0.17.0", note = "use get_files_iter() instead")]
#[inline]
pub fn get_files(&self) -> Vec<Path> {
self.state.file_paths_iter().collect()
}

/// Returns file names present in the loaded state in HashSet
#[deprecated(since = "0.17.0", note = "use get_files_iter() instead")]
pub fn get_file_set(&self) -> HashSet<Path> {
self.state.file_paths_iter().collect()
}
Expand Down Expand Up @@ -830,6 +832,10 @@ impl DeltaTable {
}

/// Return the tables configurations that are encapsulated in the DeltaTableStates currentMetaData field
#[deprecated(
since = "0.17.0",
note = "use metadata().configuration or get_state().table_config() instead"
)]
pub fn get_configurations(&self) -> Result<&HashMap<String, Option<String>>, DeltaTableError> {
Ok(self
.state
Expand Down