Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/catalog/rest/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ impl RestCatalog {
}

/// All requests and expected responses are derived from the REST catalog API spec:
/// https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml
/// <https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml>
#[async_trait]
impl Catalog for RestCatalog {
async fn list_namespaces(
Expand Down
106 changes: 94 additions & 12 deletions crates/iceberg/public-api.txt

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion crates/iceberg/src/arrow/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ impl ExtensionType for VariantExtensionType {

/// A post order arrow schema visitor.
///
/// For order of methods called, please refer to [`visit_schema`].
/// For order of methods called, please refer to the internal `visit_schema` function.
pub trait ArrowSchemaVisitor {
/// Return type of this visitor on arrow field.
type T;
Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/encryption/crypto.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use crate::{Error, ErrorKind, Result};

/// Wrapper for sensitive byte data (encryption keys, DEKs, etc.) that:
/// - Zeroizes memory on drop
/// - Redacts content in [`Debug`] and [`Display`] output
/// - Redacts content in [`Debug`](std::fmt::Debug) and [`Display`](std::fmt::Display) output
/// - Provides only `&[u8]` access via [`as_bytes()`](Self::as_bytes)
/// - Uses `Box<[u8]>` (immutable boxed slice) since key bytes never grow
///
Expand Down Expand Up @@ -217,7 +217,7 @@ impl AesGcmCipher {
/// * `aad` - Additional authenticated data (optional)
///
/// # Returns
/// The encrypted data in the format: [12-byte nonce][ciphertext][16-byte auth tag]
/// The encrypted data in the format: `[12-byte nonce][ciphertext][16-byte auth tag]`
/// This matches the Java implementation format for compatibility.
pub fn encrypt(&self, plaintext: &[u8], aad: Option<&[u8]>) -> Result<Vec<u8>> {
match self.key_size {
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/encryption/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ impl EncryptionManager {
///
/// Stores the resulting wrapped entry (and any newly created KEK) in the
/// manager's internal `encryption_keys` map. Callers persist the full set
/// at commit time via [`Self::encryption_keys`].
/// at commit time via the manager's `encryption_keys`.
///
/// Returns the `key_id` of the wrapped entry, which should be recorded on
/// the snapshot as `encryption_key_id` so readers can locate it later.
Expand Down
1 change: 0 additions & 1 deletion crates/iceberg/src/io/storage/config/azdls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ pub const ADLS_AUTHORITY_HOST: &str = "adls.authority-host";
///
/// This struct contains all the configuration options for connecting to Azure Data Lake Storage.
/// Use the builder pattern via `AzdlsConfig::builder()` to construct instances.
/// ```
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
pub struct AzdlsConfig {
/// Connection string.
Expand Down
3 changes: 1 addition & 2 deletions crates/iceberg/src/io/storage/config/gcs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//! Google Cloud Storage configuration.
//!
//! This module provides configuration constants and types for Google Cloud Storage.
//! Reference: https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
//! Reference: <https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java>

use serde::{Deserialize, Serialize};
use typed_builder::TypedBuilder;
Expand Down Expand Up @@ -52,7 +52,6 @@ pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load";
///
/// This struct contains all the configuration options for connecting to Google Cloud Storage.
/// Use the builder pattern via `GcsConfig::builder()` to construct instances.
/// ```
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
pub struct GcsConfig {
/// Google Cloud Project ID.
Expand Down
1 change: 0 additions & 1 deletion crates/iceberg/src/io/storage/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ use serde::{Deserialize, Serialize};
/// This struct contains only configuration properties without specifying
/// which storage backend to use. The storage type is determined by the
/// explicit factory selection.
/// ```
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct StorageConfig {
/// Configuration properties for the storage backend
Expand Down
1 change: 0 additions & 1 deletion crates/iceberg/src/io/storage/config/oss.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ pub const OSS_ACCESS_KEY_SECRET: &str = "oss.access-key-secret";
///
/// This struct contains all the configuration options for connecting to Alibaba Cloud OSS.
/// Use the builder pattern via `OssConfig::builder()` to construct instances.
/// ```
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
pub struct OssConfig {
/// OSS endpoint URL.
Expand Down
1 change: 0 additions & 1 deletion crates/iceberg/src/io/storage/local_fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ use crate::{Error, ErrorKind, Result};
/// - `file:///path/to/file` -> `/path/to/file`
/// - `file:/path/to/file` -> `/path/to/file`
/// - `/path/to/file` -> `/path/to/file`
/// ```
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct LocalFsStorage;

Expand Down
1 change: 0 additions & 1 deletion crates/iceberg/src/io/storage/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ use crate::{Error, ErrorKind, Result};
/// deserialized, it creates a new empty instance. This is intentional
/// because in-memory data cannot be meaningfully serialized across
/// process boundaries.
/// ```
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct MemoryStorage {
#[serde(skip, default = "default_memory_data")]
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ pub mod transaction;
pub mod transform;

mod runtime;
pub use runtime::{Runtime, RuntimeHandle};
pub use runtime::{JoinHandle, Runtime, RuntimeHandle};

pub mod arrow;
pub(crate) mod delete_file_index;
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/puffin/blob.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ pub struct Blob {

impl Blob {
#[inline]
/// See blob types: https://iceberg.apache.org/puffin-spec/#blob-types
/// See blob types: <https://iceberg.apache.org/puffin-spec/#blob-types>
pub fn blob_type(&self) -> &str {
&self.r#type
}
Expand Down
6 changes: 3 additions & 3 deletions crates/iceberg/src/puffin/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use crate::{Error, ErrorKind, Result};
pub const CREATED_BY_PROPERTY: &str = "created-by";

/// Metadata about a blob.
/// For more information, see: https://iceberg.apache.org/puffin-spec/#blobmetadata
/// For more information, see: <https://iceberg.apache.org/puffin-spec/#blobmetadata>
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct BlobMetadata {
Expand All @@ -49,7 +49,7 @@ pub struct BlobMetadata {

impl BlobMetadata {
#[inline]
/// See blob types: https://iceberg.apache.org/puffin-spec/#blob-types
/// See blob types: <https://iceberg.apache.org/puffin-spec/#blob-types>
pub fn blob_type(&self) -> &str {
&self.r#type
}
Expand Down Expand Up @@ -129,7 +129,7 @@ impl Flag {

/// Metadata about a puffin file.
///
/// For more information, see: https://iceberg.apache.org/puffin-spec/#filemetadata
/// For more information, see: <https://iceberg.apache.org/puffin-spec/#filemetadata>
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]
pub struct FileMetadata {
pub(crate) blobs: Vec<BlobMetadata>,
Expand Down
26 changes: 13 additions & 13 deletions crates/iceberg/src/runtime/mod.rs
Comment thread
JosephLenton marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@ use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};

use tokio::task;
use tokio::task::JoinHandle as TokioJoinHandle;

use crate::{Error, ErrorKind, Result};

/// Wrapper around tokio's `JoinHandle` that converts task failures into
/// [`iceberg::Error`].
/// Wrapper around tokio's [`TokioJoinHandle`] that converts task failures into
/// [`crate::Error`].
///
/// Tokio's `JoinHandle<T>` resolves to `Result<T, JoinError>`, where a
/// `JoinError` means the task either panicked or was cancelled (typically from
/// runtime shutdown or `abort`). Both are surfaced here as
/// `ErrorKind::Unexpected` with the original `JoinError` preserved as the
/// source.
pub struct JoinHandle<T>(task::JoinHandle<T>);
/// [`tokio::task::JoinError`] means the task either panicked or was cancelled
/// (typically from runtime shutdown or `abort`).
/// Both are surfaced here as `ErrorKind::Unexpected` with the original
/// [`tokio::task::JoinError`] preserved as the source.
pub struct JoinHandle<T>(TokioJoinHandle<T>);

impl<T> Unpin for JoinHandle<T> {}

Expand All @@ -52,8 +52,8 @@ impl<T: Send + 'static> Future for JoinHandle<T> {
///
/// Wraps a [`tokio::runtime::Handle`], which is cheap to clone. The caller is
/// responsible for keeping the underlying runtime alive while this handle is
/// in use; spawning on a shut-down runtime will surface as a `JoinError` via
/// [`JoinHandle`].
/// in use; spawning on a shut-down runtime will surface as a [`tokio::task::JoinError`]
/// via the returned [`JoinHandle`].
#[derive(Clone)]
pub struct RuntimeHandle {
handle: tokio::runtime::Handle,
Expand Down Expand Up @@ -98,10 +98,10 @@ impl RuntimeHandle {
///
/// # Lifetime
///
/// A `Runtime` stores only `tokio::runtime::Handle`s (weak references). The
/// caller owns the tokio runtime's lifetime. If the underlying runtime is
/// A `Runtime` stores only `tokio::runtime::Handle`s (weak references).
/// The caller owns the tokio runtime's lifetime. If the underlying runtime is
/// dropped while iceberg is still using it, subsequent spawns will surface as
/// task cancellation errors via [`JoinHandle`].
/// task cancellation errors via the returned `JoinHandle`.
///
/// Cloning is cheap.
#[derive(Clone)]
Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/spec/datatypes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ impl Type {
}
}

/// Return max precision for decimal given [`num_bytes`] bytes.
/// Return max precision for decimal given `num_bytes` bytes.
#[inline(always)]
pub fn decimal_max_precision(num_bytes: u32) -> Result<u32> {
ensure_data_valid!(
Expand All @@ -177,7 +177,7 @@ impl Type {
Ok(MAX_PRECISION[num_bytes as usize - 1])
}

/// Returns minimum bytes required for decimal with [`precision`].
/// Returns minimum bytes required for decimal with `precision`.
#[inline(always)]
pub fn decimal_required_bytes(precision: u32) -> Result<u32> {
ensure_data_valid!(
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/spec/manifest/data_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ pub struct DataFile {
/// field id: 142
///
/// The _row_id for the first row in the data file.
/// For more details, refer to https://github.com/apache/iceberg/blob/main/format/spec.md#first-row-id-inheritance
/// For more details, refer to <https://github.com/apache/iceberg/blob/main/format/spec.md#first-row-id-inheritance>
#[builder(default)]
pub(crate) first_row_id: Option<i64>,
/// This field is not included in spec. It is just store in memory representation used
Expand Down
8 changes: 4 additions & 4 deletions crates/iceberg/src/spec/schema/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -364,25 +364,25 @@ impl Schema {
.and_then(|id| self.field_by_id(*id))
}

/// Returns [`highest_field_id`].
/// Returns the highest field ID assigned in this schema.
#[inline]
pub fn highest_field_id(&self) -> i32 {
self.highest_field_id
}

/// Returns [`schema_id`].
/// Returns the Schema's ID.
#[inline]
pub fn schema_id(&self) -> SchemaId {
self.schema_id
}

/// Returns [`r#struct`].
/// Returns the type for the struct.
#[inline]
pub fn as_struct(&self) -> &StructType {
&self.r#struct
}

/// Returns [`identifier_field_ids`].
/// Returns the IDs of the fields.
#[inline]
pub fn identifier_field_ids(&self) -> impl ExactSizeIterator<Item = i32> + '_ {
self.identifier_field_ids.iter().copied()
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/spec/table_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1605,7 +1605,7 @@ pub struct SnapshotLog {
}

impl SnapshotLog {
/// Returns the last updated timestamp as a DateTime<Utc> with millisecond precision
/// Returns the last updated timestamp as a [`DateTime<Utc>`] with millisecond precision
pub fn timestamp(self) -> Result<DateTime<Utc>> {
timestamp_ms_to_utc(self.timestamp_ms)
}
Expand Down
10 changes: 5 additions & 5 deletions crates/iceberg/src/spec/table_properties.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,12 +252,12 @@ pub struct TableProperties {
/// Deprecated table property for data file write location.
///
/// Property will be removed at a later date.
/// Superseded by [write_data_location].
/// Superseded by [`TableProperties::write_data_location`].
pub write_folder_storage_location: Option<String>,
/// Deprecated table property for data file write location for object storage location generator.
///
/// Property will be removed at a later date.
/// Superseded by [write_data_location].
/// Superseded by [`TableProperties::write_data_location`].
pub write_object_storage_location: Option<String>,
/// Whether partition values are included in object storage paths.
pub write_object_storage_partitioned_paths: bool,
Expand Down Expand Up @@ -437,7 +437,7 @@ impl TableProperties {
pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024;

/// Property key for the master key id used to encrypt the table's manifest
/// list and data files as defined in https://iceberg.apache.org/docs/nightly/encryption/.
/// list and data files as defined in <https://iceberg.apache.org/docs/nightly/encryption/>.
pub const PROPERTY_ENCRYPTION_KEY_ID: &str = "encryption.key-id";

/// Property key for the encryption data encryption key (DEK) length in bytes.
Expand All @@ -446,14 +446,14 @@ impl TableProperties {
pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16;
/// Property key for the base directory for data files
pub const PROPERTY_WRITE_DATA_LOCATION: &str = "write.data.path";
/// Property key for deprecated [write_folder_storage_location]
/// Property key for deprecated [`TableProperties::write_folder_storage_location`]
pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path";
/// Property key for deprecated object storage path, kept as a fallback for compatibility.
pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &str = "write.object-storage.path";
/// Property key for controlling whether partition values are included in object storage paths.
pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &str =
"write.object-storage.partitioned-paths";
/// Default value for [PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS]
/// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS`]
pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true;
}

Expand Down
4 changes: 2 additions & 2 deletions crates/iceberg/src/spec/values/datum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -822,7 +822,7 @@ impl Datum {
Self::timestamp_micros(dt.and_utc().timestamp_micros())
}

/// Parse a timestamp in [`%Y-%m-%dT%H:%M:%S%.f`] format.
/// Parse a timestamp in `%Y-%m-%dT%H:%M:%S%.f` format.
///
/// See [`NaiveDateTime::from_str`].
///
Expand Down Expand Up @@ -1240,7 +1240,7 @@ impl Datum {
/// Returns a human-readable string representation of this literal.
///
/// For string literals, this returns the raw string value without quotes.
/// For all other literals, it falls back to [`to_string()`].
/// For all other literals, it falls back to [`to_string()`](ToString::to_string).
pub fn to_human_string(&self) -> String {
match self.literal() {
PrimitiveLiteral::String(s) => s.to_string(),
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/spec/view_metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ impl ViewVersionLog {
self.timestamp_ms
}

/// Returns the last updated timestamp as a DateTime<Utc> with millisecond precision.
/// Returns the last updated timestamp as a `DateTime<Utc>` with millisecond precision.
pub fn timestamp(&self) -> Result<DateTime<Utc>> {
timestamp_ms_to_utc(self.timestamp_ms)
}
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/test_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use crate::spec::TableMetadata;
use crate::table::Table;

/// Returns a process-wide [`Runtime`] suitable for tests that need to construct
/// a [`Table`](crate::table::Table) outside a tokio context.
/// a [`Table`] outside a tokio context.
///
/// The returned [`Runtime`] wraps a single shared multi-thread tokio runtime
/// that is lazily built on first call and lives until process exit. Cloning is
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/transaction/action.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub(crate) type BoxedTransactionAction = Arc<dyn TransactionAction>;
/// Each action is responsible for generating the updates and requirements needed
/// to modify the table metadata.
#[async_trait]
pub(crate) trait TransactionAction: AsAny + Sync + Send {
pub trait TransactionAction: AsAny + Sync + Send {
/// Commits this action against the provided table and returns the resulting updates.
/// NOTE: This function is intended for internal use only and should not be called directly by users.
///
Expand Down
16 changes: 8 additions & 8 deletions crates/iceberg/src/transaction/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,14 @@ use crate::error::Result;
use crate::spec::TableProperties;
use crate::table::Table;
use crate::transaction::action::BoxedTransactionAction;
use crate::transaction::append::FastAppendAction;
use crate::transaction::expire_snapshots::ExpireSnapshotsAction;
use crate::transaction::sort_order::ReplaceSortOrderAction;
use crate::transaction::update_location::UpdateLocationAction;
use crate::transaction::update_properties::UpdatePropertiesAction;
use crate::transaction::update_schema::UpdateSchemaAction;
use crate::transaction::update_statistics::UpdateStatisticsAction;
use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction;
pub use crate::transaction::append::FastAppendAction;
pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction;
pub use crate::transaction::sort_order::ReplaceSortOrderAction;
pub use crate::transaction::update_location::UpdateLocationAction;
pub use crate::transaction::update_properties::UpdatePropertiesAction;
pub use crate::transaction::update_schema::UpdateSchemaAction;
pub use crate::transaction::update_statistics::UpdateStatisticsAction;
pub use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction;
use crate::{Catalog, Error, ErrorKind, TableCommit, TableRequirement, TableUpdate};

/// Table transaction.
Expand Down
2 changes: 1 addition & 1 deletion crates/iceberg/src/transaction/sort_order.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ pub struct ReplaceSortOrderAction {
}

impl ReplaceSortOrderAction {
pub fn new() -> Self {
pub(crate) fn new() -> Self {
ReplaceSortOrderAction {
pending_sort_fields: vec![],
}
Expand Down
Loading