Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 0 additions & 28 deletions crates/iceberg/public-api.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2757,34 +2757,6 @@ pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::Table
impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder
pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
pub struct iceberg::spec::TableProperties
pub iceberg::spec::TableProperties::cdc_enabled: bool

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we are removing these public API, should the PR title have a refactor![...]?

pub iceberg::spec::TableProperties::cdc_max_chunk_size: usize
pub iceberg::spec::TableProperties::cdc_min_chunk_size: usize
pub iceberg::spec::TableProperties::cdc_norm_level: i32
pub iceberg::spec::TableProperties::commit_max_retry_wait_ms: u64
pub iceberg::spec::TableProperties::commit_min_retry_wait_ms: u64
pub iceberg::spec::TableProperties::commit_num_retries: usize
pub iceberg::spec::TableProperties::commit_total_retry_timeout_ms: u64
pub iceberg::spec::TableProperties::encryption_data_key_length: usize
pub iceberg::spec::TableProperties::encryption_key_id: core::option::Option<alloc::string::String>
pub iceberg::spec::TableProperties::gc_enabled: bool
pub iceberg::spec::TableProperties::max_ref_age_ms: i64
pub iceberg::spec::TableProperties::max_snapshot_age_ms: i64
pub iceberg::spec::TableProperties::metadata_compression_codec: iceberg::compression::CompressionCodec
pub iceberg::spec::TableProperties::min_snapshots_to_keep: usize
pub iceberg::spec::TableProperties::parquet_compression_codec: iceberg::compression::CompressionCodec
pub iceberg::spec::TableProperties::parquet_dict_size_bytes: usize
pub iceberg::spec::TableProperties::parquet_page_row_limit: usize
pub iceberg::spec::TableProperties::parquet_page_size_bytes: usize
pub iceberg::spec::TableProperties::parquet_row_group_size_bytes: usize
pub iceberg::spec::TableProperties::write_data_location: core::option::Option<alloc::string::String>
pub iceberg::spec::TableProperties::write_datafusion_fanout_enabled: bool
pub iceberg::spec::TableProperties::write_folder_storage_location: core::option::Option<alloc::string::String>
pub iceberg::spec::TableProperties::write_format_default: alloc::string::String
pub iceberg::spec::TableProperties::write_metadata_path: core::option::Option<alloc::string::String>
pub iceberg::spec::TableProperties::write_object_storage_location: core::option::Option<alloc::string::String>
pub iceberg::spec::TableProperties::write_object_storage_partitioned_paths: bool
pub iceberg::spec::TableProperties::write_target_file_size_bytes: usize
impl iceberg::spec::TableProperties
pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str
pub const iceberg::spec::TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/catalog/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ pub async fn drop_table_data(table_info: &Table) -> Result<()> {
}

// Delete data files only if gc.enabled is true, to avoid corrupting shared tables
if metadata.table_properties()?.gc_enabled {
if metadata.table_properties()?.gc_enabled() {
delete_data_files(io, &manifests_to_delete).await?;
}

Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/encryption/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ impl EncryptionManager {
}

let table_properties = metadata.table_properties()?;
let Some(table_key_id) = table_properties.encryption_key_id else {
let Some(table_key_id) = table_properties.encryption_key_id().clone() else {
if kms_client.is_some() {
tracing::warn!(
"KeyManagementClient provided but table does not have encryption.key-id set"
Expand All @@ -140,7 +140,7 @@ impl EncryptionManager {
.table_key_id(table_key_id)
.encryption_keys(metadata.encryption_keys.clone())
.key_size(AesKeySize::from_key_length(
table_properties.encryption_data_key_length,
table_properties.encryption_data_key_length(),
)?)
.build();
Ok(Some(Arc::new(em)))
Expand Down
11 changes: 6 additions & 5 deletions crates/iceberg/src/spec/table_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,8 @@ impl TableMetadata {
pub fn metadata_location(&self) -> Result<String> {
Ok(self
.table_properties()?
.write_metadata_path
.write_metadata_path()
.clone()
.unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME)))
}

Expand Down Expand Up @@ -4043,11 +4044,11 @@ mod tests {
let props = metadata.table_properties().unwrap();

assert_eq!(
props.commit_num_retries,
props.commit_num_retries(),
TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
);
assert_eq!(
props.write_target_file_size_bytes,
props.write_target_file_size_bytes(),
TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
);
}
Expand Down Expand Up @@ -4089,8 +4090,8 @@ mod tests {

let props = metadata.table_properties().unwrap();

assert_eq!(props.commit_num_retries, 10);
assert_eq!(props.write_target_file_size_bytes, 1024);
assert_eq!(props.commit_num_retries(), 10);
assert_eq!(props.write_target_file_size_bytes(), 1024);
}

#[test]
Expand Down
56 changes: 28 additions & 28 deletions crates/iceberg/src/spec/table_properties.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,42 +153,42 @@ pub struct TableProperties {
default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT,
getter
)]
pub commit_num_retries: usize,
commit_num_retries: usize,
/// The minimum wait time between retries.
#[property(
key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS,
default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT,
getter
)]
pub commit_min_retry_wait_ms: u64,
commit_min_retry_wait_ms: u64,
/// The maximum wait time between retries.
#[property(
key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS,
default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT,
getter
)]
pub commit_max_retry_wait_ms: u64,
commit_max_retry_wait_ms: u64,
/// The total timeout for commit retries.
#[property(
key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS,
default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT,
getter
)]
pub commit_total_retry_timeout_ms: u64,
commit_total_retry_timeout_ms: u64,
/// The default format for files.
#[property(
key = Self::PROPERTY_DEFAULT_FILE_FORMAT,
default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT,
getter
)]
pub write_format_default: String,
write_format_default: String,
/// The target file size for files.
#[property(
key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES,
default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT,
getter
)]
pub write_target_file_size_bytes: usize,
write_target_file_size_bytes: usize,
/// Base directory for metadata files (manifests, manifest lists), with any
/// trailing slash trimmed. `None` if `write.metadata.path` is not set.
#[property(
Expand All @@ -197,80 +197,80 @@ pub struct TableProperties {
parse_with = parse_location_property,
getter
)]
pub write_metadata_path: Option<String>,
write_metadata_path: Option<String>,
/// Compression codec for metadata files (JSON)
#[property(
key = Self::PROPERTY_METADATA_COMPRESSION_CODEC,
default = CompressionCodec::None,
parse_with = parse_metadata_compression,
getter
)]
pub metadata_compression_codec: CompressionCodec,
metadata_compression_codec: CompressionCodec,
/// Whether to use `FanoutWriter` for partitioned tables.
#[property(
key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED,
default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT,
getter
)]
pub write_datafusion_fanout_enabled: bool,
write_datafusion_fanout_enabled: bool,
/// Whether garbage collection is enabled on drop.
/// When `false`, data files will not be deleted when a table is dropped.
#[property(
key = Self::PROPERTY_GC_ENABLED,
default = Self::PROPERTY_GC_ENABLED_DEFAULT,
getter
)]
pub gc_enabled: bool,
gc_enabled: bool,
/// Default maximum age of a snapshot to keep when expiring snapshots.
#[property(
key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS,
default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT,
getter
)]
pub max_snapshot_age_ms: i64,
max_snapshot_age_ms: i64,
/// Default minimum number of snapshots to keep per branch when expiring snapshots.
#[property(
key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP,
default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT,
getter
)]
pub min_snapshots_to_keep: usize,
min_snapshots_to_keep: usize,
/// Default maximum age of a snapshot reference to keep when expiring snapshots.
#[property(
key = Self::PROPERTY_MAX_REF_AGE_MS,
default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT,
getter
)]
pub max_ref_age_ms: i64,
max_ref_age_ms: i64,
/// Whether content-defined chunking is enabled.
/// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`.
#[property(
key = Self::PROPERTY_PARQUET_CDC_ENABLED,
default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT,
getter
)]
pub cdc_enabled: bool,
cdc_enabled: bool,
/// Content-defined chunking minimum chunk size in bytes.
#[property(
key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE,
default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT,
getter
)]
pub cdc_min_chunk_size: usize,
cdc_min_chunk_size: usize,
/// Content-defined chunking maximum chunk size in bytes.
#[property(
key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE,
default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT,
getter
)]
pub cdc_max_chunk_size: usize,
cdc_max_chunk_size: usize,
/// Content-defined chunking normalization level (gearhash bit adjustment).
#[property(
key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL,
default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT,
getter
)]
pub cdc_norm_level: i32,
cdc_norm_level: i32,
/// Parquet compression codec for data files, with the resolved compression
/// level folded in (from `write.parquet.compression-level`, or the codec's
/// default when unset).
Expand All @@ -281,57 +281,57 @@ pub struct TableProperties {
parse_properties_with = parse_parquet_compression,
getter
)]
pub parquet_compression_codec: CompressionCodec,
parquet_compression_codec: CompressionCodec,
/// Approximate maximum Parquet row group size in bytes.
#[property(
key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES,
default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT,
getter
)]
pub parquet_row_group_size_bytes: usize,
parquet_row_group_size_bytes: usize,
/// Approximate maximum Parquet data page size in bytes.
#[property(
key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES,
default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT,
getter
)]
pub parquet_page_size_bytes: usize,
parquet_page_size_bytes: usize,
/// Maximum number of rows per Parquet data page.
#[property(
key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT,
default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT,
getter
)]
pub parquet_page_row_limit: usize,
parquet_page_row_limit: usize,
/// Approximate maximum Parquet dictionary page size in bytes.
#[property(
key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES,
default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT,
getter
)]
pub parquet_dict_size_bytes: usize,
parquet_dict_size_bytes: usize,
/// The master key id used to encrypt this table's manifest list and data
/// files. `None` if `encryption.key-id` is not set.
#[property(
key = Self::PROPERTY_ENCRYPTION_KEY_ID,
default = None,
getter
)]
pub encryption_key_id: Option<String>,
encryption_key_id: Option<String>,
/// The encryption data encryption key length in bytes.
#[property(
key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH,
default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT,
getter
)]
pub encryption_data_key_length: usize,
encryption_data_key_length: usize,
/// Base directory for data files
#[property(
key = Self::PROPERTY_WRITE_DATA_LOCATION,
default = None,
getter
)]
pub write_data_location: Option<String>,
write_data_location: Option<String>,
/// Deprecated table property for data file write location.
///
/// Property will be removed at a later date.
Expand All @@ -341,7 +341,7 @@ pub struct TableProperties {
default = None,
getter
)]
pub write_folder_storage_location: Option<String>,
write_folder_storage_location: Option<String>,
/// Deprecated table property for data file write location for object storage location generator.
///
/// Property will be removed at a later date.
Expand All @@ -351,14 +351,14 @@ pub struct TableProperties {
default = None,
getter
)]
pub write_object_storage_location: Option<String>,
write_object_storage_location: Option<String>,
/// Whether partition values are included in object storage paths.
#[property(
key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS,
default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT,
getter
)]
pub write_object_storage_partitioned_paths: bool,
write_object_storage_partitioned_paths: bool,
}

impl TableProperties {
Expand Down
10 changes: 6 additions & 4 deletions crates/iceberg/src/transaction/expire_snapshots.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,10 @@ impl ExpireSnapshotsAction {
// days) the age path always runs, so even an explicit-id-only call applies the default cutoff.
let default_cutoff = self
.older_than_ms
.unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms));
let default_min_to_keep = self.retain_last.unwrap_or(properties.min_snapshots_to_keep);
.unwrap_or_else(|| now.saturating_sub(properties.max_snapshot_age_ms()));
let default_min_to_keep = self
.retain_last
.unwrap_or(properties.min_snapshots_to_keep());

// Ref aging: `main` is always kept; any other ref whose head is older than its
// `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's
Expand All @@ -125,7 +127,7 @@ impl ExpireSnapshotsAction {
let mut retained_refs: Vec<&SnapshotReference> = vec![];
for (ref_name, snapshot_ref) in &metadata.refs {
if ref_name == MAIN_BRANCH
|| !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms)
|| !Self::ref_aged_out(metadata, snapshot_ref, now, properties.max_ref_age_ms())
{
retained_refs.push(snapshot_ref);
} else {
Expand Down Expand Up @@ -302,7 +304,7 @@ impl TransactionAction for ExpireSnapshotsAction {
let properties = metadata.table_properties()?;

// Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too).
if !properties.gc_enabled {
if !properties.gc_enabled() {
return Err(Error::new(
ErrorKind::DataInvalid,
"Cannot expire snapshots: gc.enabled is false",
Expand Down
10 changes: 5 additions & 5 deletions crates/iceberg/src/transaction/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ impl Transaction {
let table_props = self.table.metadata().table_properties()?;

// TODO(https://github.com/apache/iceberg-rust/issues/2034): remove once encrypted writes are supported
if table_props.encryption_key_id.is_some() {
if table_props.encryption_key_id().is_some() {
return Err(Error::new(
ErrorKind::FeatureUnsupported,
"Cannot commit to an encrypted table: encrypted writes are not yet supported",
Expand All @@ -205,12 +205,12 @@ impl Transaction {

fn build_backoff(props: TableProperties) -> Result<ExponentialBackoff> {
Ok(ExponentialBuilder::new()
.with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms))
.with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms))
.with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms()))
.with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms()))
.with_total_delay(Some(Duration::from_millis(
props.commit_total_retry_timeout_ms,
props.commit_total_retry_timeout_ms(),
)))
.with_max_times(props.commit_num_retries)
.with_max_times(props.commit_num_retries())
.with_factor(2.0)
.build())
}
Expand Down
Loading
Loading