Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
364 changes: 364 additions & 0 deletions nexus/db-queries/src/db/datastore/external_subnet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@ use diesel::JoinOnDsl as _;
use diesel::NullableExpressionMethods as _;
use diesel::QueryDsl as _;
use diesel::SelectableHelper as _;
use diesel::define_sql_function;
use diesel::result::DatabaseErrorKind;
use diesel::result::Error as DieselError;
use diesel::sql_types::Double;
use diesel::sql_types::Nullable;
use dropshot::PaginationOrder;
use nexus_auth::authz;
use nexus_auth::authz::SUBNET_POOL_LIST;
Expand Down Expand Up @@ -126,6 +129,8 @@ impl From<AttachedSubnetDetails> for AttachedSubnet {
}
}

define_sql_function!(fn coalesce(x: Nullable<Double>, y: Double) -> Double);

impl DataStore {
/// Lookup a Subnet Pool by name or ID.
pub fn lookup_subnet_pool<'a>(
Expand Down Expand Up @@ -638,6 +643,92 @@ impl DataStore {
Ok(())
}

// === Subnet Pool Utilization ===

/// Return (allocated, capacity) as f64 address counts for a subnet pool.
///
/// Both values are computed entirely in SQL using
/// `SUM(pow(2, bits - prefix_len))` over the member and allocated subnet
/// tables. Because CockroachDB cannot do 128-bit integer arithmetic on
/// inet values, we use FLOAT8 (f64), which is also the type exposed in
/// the API response. For IPv6 subnets this loses precision in the low
/// bits, but the API already accepts that tradeoff.
pub async fn subnet_pool_utilization(
&self,
opctx: &OpContext,
authz_pool: &authz::SubnetPool,
) -> Result<(f64, f64), Error> {
use diesel::dsl::{sql, sum};
use diesel::sql_types::Double as SqlDouble;
use nexus_db_schema::schema::external_subnet;
use nexus_db_schema::schema::subnet_pool_member;

// SQL expression for the number of addresses in a subnet.
// Diesel has no native inet function support, so this stays as
// a raw SQL fragment; everything else uses the DSL.
const SUBNET_SIZE_SQL: &str = "pow(2::FLOAT8, \
(CASE WHEN family(subnet) = 4 THEN 32 ELSE 128 END \
- masklen(subnet))::FLOAT8)";

opctx.authorize(authz::Action::Read, authz_pool).await?;
opctx.authorize(authz::Action::ListChildren, authz_pool).await?;
let conn = self.pool_connection_authorized(opctx).await?;
let pool_id = to_db_typed_uuid(authz_pool.id());

let capacity_subq = subnet_pool_member::table
.filter(subnet_pool_member::subnet_pool_id.eq(pool_id))
.filter(subnet_pool_member::time_deleted.is_null())
.select(sum(sql::<SqlDouble>(SUBNET_SIZE_SQL)))
.single_value();

let allocated_subq = external_subnet::table
.filter(external_subnet::subnet_pool_id.eq(pool_id))
.filter(external_subnet::time_deleted.is_null())
.select(sum(sql::<SqlDouble>(SUBNET_SIZE_SQL)))
.single_value();

let (capacity, allocated) = diesel::select((
coalesce(capacity_subq, 0.0),
coalesce(allocated_subq, 0.0),
))
.get_result_async::<(f64, f64)>(&*conn)
.await
.map_err(|e| match &e {
DieselError::NotFound => public_error_from_diesel(
e,
ErrorHandler::NotFoundByResource(authz_pool),
),
_ => public_error_from_diesel(e, ErrorHandler::Server),
})?;

Ok((allocated, capacity))
}

/// Return the total capacity (in addresses) of the provided Subnet Pool.
#[cfg(test)]
async fn subnet_pool_total_capacity(
&self,
opctx: &OpContext,
authz_pool: &authz::SubnetPool,
) -> Result<f64, Error> {
let (_, capacity) =
self.subnet_pool_utilization(opctx, authz_pool).await?;
Ok(capacity)
}

/// Return the total number of allocated addresses in the provided Subnet
/// Pool.
#[cfg(test)]
async fn subnet_pool_allocated_count(
&self,
opctx: &OpContext,
authz_pool: &authz::SubnetPool,
) -> Result<f64, Error> {
let (allocated, _) =
self.subnet_pool_utilization(opctx, authz_pool).await?;
Ok(allocated)
}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Compare to IP pool utilization:

/// Return the number of IPs allocated from and the capacity of the provided
/// IP Pool.
pub async fn ip_pool_utilization(
&self,
opctx: &OpContext,
authz_pool: &authz::IpPool,
) -> Result<(i64, u128), Error> {
opctx.authorize(authz::Action::Read, authz_pool).await?;
opctx.authorize(authz::Action::ListChildren, authz_pool).await?;
let conn = self.pool_connection_authorized(opctx).await?;
let (allocated, ranges) = self
.transaction_retry_wrapper("ip_pool_utilization")
.transaction(&conn, |conn| async move {
let allocated = self
.ip_pool_allocated_count_on_connection(&conn, authz_pool)
.await?;
let ranges = self
.ip_pool_list_ranges_batched_on_connection(
&conn, authz_pool,
)
.await?;
Ok((allocated, ranges))
})
.await
.map_err(|e| match &e {
DieselError::NotFound => public_error_from_diesel(
e,
ErrorHandler::NotFoundByResource(authz_pool),
),
_ => public_error_from_diesel(e, ErrorHandler::Server),
})?;
let capacity = Self::accumulate_ip_range_sizes(ranges)?;
Ok((allocated, capacity))
}
/// Return the total number of IPs allocated from the provided pool.
#[cfg(test)]
async fn ip_pool_allocated_count(
&self,
opctx: &OpContext,
authz_pool: &authz::IpPool,
) -> Result<i64, Error> {
opctx.authorize(authz::Action::Read, authz_pool).await?;
let conn = self.pool_connection_authorized(opctx).await?;
self.ip_pool_allocated_count_on_connection(&conn, authz_pool)
.await
.map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
}
async fn ip_pool_allocated_count_on_connection(
&self,
conn: &async_bb8_diesel::Connection<DbConnection>,
authz_pool: &authz::IpPool,
) -> Result<i64, DieselError> {
use nexus_db_schema::schema::external_ip;
external_ip::table
.filter(external_ip::ip_pool_id.eq(authz_pool.id()))
.filter(external_ip::time_deleted.is_null())
.select(count(external_ip::ip).aggregate_distinct())
.first_async::<i64>(conn)
.await
}
/// Return the total capacity of the provided pool.
#[cfg(test)]
async fn ip_pool_total_capacity(
&self,
opctx: &OpContext,
authz_pool: &authz::IpPool,
) -> Result<u128, Error> {
opctx.authorize(authz::Action::Read, authz_pool).await?;
opctx.authorize(authz::Action::ListChildren, authz_pool).await?;
let conn = self.pool_connection_authorized(opctx).await?;
self.ip_pool_list_ranges_batched_on_connection(&conn, authz_pool)
.await
.map_err(|e| {
public_error_from_diesel(
e,
ErrorHandler::NotFoundByResource(authz_pool),
)
})
.and_then(Self::accumulate_ip_range_sizes)
}
async fn ip_pool_list_ranges_batched_on_connection(
&self,
conn: &async_bb8_diesel::Connection<DbConnection>,
authz_pool: &authz::IpPool,
) -> Result<Vec<(IpNetwork, IpNetwork)>, DieselError> {
use nexus_db_schema::schema::ip_pool_range;
ip_pool_range::table
.filter(ip_pool_range::ip_pool_id.eq(authz_pool.id()))
.filter(ip_pool_range::time_deleted.is_null())
.select((ip_pool_range::first_address, ip_pool_range::last_address))
// This is a rare unpaginated DB query, which means we are
// vulnerable to a resource exhaustion attack in which someone
// creates a very large number of ranges in order to make this
// query slow. In order to mitigate that, we limit the query to the
// (current) max allowed page size, effectively making this query
// exactly as vulnerable as if it were paginated. If there are more
// than 10,000 ranges in a pool, we will undercount, but I have a
// hard time seeing that as a practical problem.
.limit(10000)
.get_results_async::<(IpNetwork, IpNetwork)>(conn)
.await
}
fn accumulate_ip_range_sizes(
ranges: Vec<(IpNetwork, IpNetwork)>,
) -> Result<u128, Error> {
let mut count: u128 = 0;
for range in ranges.into_iter() {
let first = range.0.ip();
let last = range.1.ip();
let r = IpRange::try_from((first, last))
.map_err(|e| Error::internal_error(e.as_str()))?;
match r {
IpRange::V4(r) => count += u128::from(r.len()),
IpRange::V6(r) => count += r.len(),
}
}
Ok(count)
}

/// Create an External Subnet.
pub async fn create_external_subnet(
&self,
Expand Down Expand Up @@ -3941,4 +4032,277 @@ mod tests {
db.terminate().await;
logctx.cleanup_successful();
}

#[tokio::test]
async fn test_ipv4_subnet_pool_utilization() {
let logctx = dev::test_setup_log("test_ipv4_subnet_pool_utilization");
let db = TestDatabase::new_with_datastore(&logctx.log).await;
let (opctx, datastore) = (db.opctx(), db.datastore());

let authz_silo = authz::Silo::new(
authz::FLEET,
DEFAULT_SILO_ID,
LookupType::ById(DEFAULT_SILO_ID),
);
let (authz_project, _db_project) = datastore
.project_create(
opctx,
Project::new(
DEFAULT_SILO_ID,
ProjectCreate {
identity: IdentityMetadataCreateParams {
name: "my-project".parse().unwrap(),
description: String::new(),
},
},
),
)
.await
.expect("able to create a project");

// Create a subnet pool.
let params = SubnetPoolCreate {
identity: IdentityMetadataCreateParams {
name: "my-pool".parse().unwrap(),
description: String::new(),
},
ip_version: IpVersion::V4,
};
let db_pool = datastore
.create_subnet_pool(opctx, params)
.await
.expect("able to create subnet pool");
let authz_pool = authz::SubnetPool::new(
authz::FLEET,
db_pool.identity.id.into(),
LookupType::ById(db_pool.identity.id.into_untyped_uuid()),
);

// Capacity is 0 because there are no members.
let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(capacity, 0.0);

// Link the pool to the silo so we can allocate external subnets.
datastore
.link_subnet_pool_to_silo(opctx, &authz_pool, &authz_silo, true)
.await
.expect("able to link pool to silo");

// Add a /24 member (256 addresses).
let member_subnet: oxnet::IpNet = "10.0.0.0/24".parse().unwrap();
datastore
.add_subnet_pool_member(
opctx,
&authz_pool,
&db_pool,
&SubnetPoolMemberAdd {
subnet: member_subnet,
min_prefix_length: Some(24),
max_prefix_length: Some(28),
},
)
.await
.expect("able to add member");

// Capacity is now 256.
let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(capacity, 256.0);

// No subnets allocated yet.
let allocated = datastore
.subnet_pool_allocated_count(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(allocated, 0.0);

// Allocate a /28 (16 addresses).
datastore
.create_external_subnet(
opctx,
&DEFAULT_SILO_ID,
&authz_project,
ExternalSubnetCreate {
identity: IdentityMetadataCreateParams {
name: "my-subnet".parse().unwrap(),
description: String::new(),
},
allocator: ExternalSubnetAllocator::Explicit {
subnet: "10.0.0.0/28".parse().unwrap(),
},
},
)
.await
.expect("able to create external subnet");

// Allocated is now 16.
let allocated = datastore
.subnet_pool_allocated_count(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(allocated, 16.0);

// Capacity is unchanged.
let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(capacity, 256.0);

db.terminate().await;
logctx.cleanup_successful();
}

#[tokio::test]
async fn test_ipv6_subnet_pool_utilization() {
let logctx = dev::test_setup_log("test_ipv6_subnet_pool_utilization");
let db = TestDatabase::new_with_datastore(&logctx.log).await;
let (opctx, datastore) = (db.opctx(), db.datastore());

let authz_silo = authz::Silo::new(
authz::FLEET,
DEFAULT_SILO_ID,
LookupType::ById(DEFAULT_SILO_ID),
);
let (authz_project, _db_project) = datastore
.project_create(
opctx,
Project::new(
DEFAULT_SILO_ID,
ProjectCreate {
identity: IdentityMetadataCreateParams {
name: "my-project".parse().unwrap(),
description: String::new(),
},
},
),
)
.await
.expect("able to create a project");

// Create an IPv6 subnet pool.
let params = SubnetPoolCreate {
identity: IdentityMetadataCreateParams {
name: "my-pool".parse().unwrap(),
description: String::new(),
},
ip_version: IpVersion::V6,
};
let db_pool = datastore
.create_subnet_pool(opctx, params)
.await
.expect("able to create subnet pool");
let authz_pool = authz::SubnetPool::new(
authz::FLEET,
db_pool.identity.id.into(),
LookupType::ById(db_pool.identity.id.into_untyped_uuid()),
);

// Link the pool to the silo.
datastore
.link_subnet_pool_to_silo(opctx, &authz_pool, &authz_silo, true)
.await
.expect("able to link pool to silo");

// Capacity starts at 0.
let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(capacity, 0.0);

// Add a /48 member (2^80 addresses).
let member_subnet: oxnet::IpNet = "2001:db8:1::/48".parse().unwrap();
datastore
.add_subnet_pool_member(
opctx,
&authz_pool,
&db_pool,
&SubnetPoolMemberAdd {
subnet: member_subnet,
min_prefix_length: Some(48),
max_prefix_length: Some(64),
},
)
.await
.expect("able to add member");

let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
// 2^80 is exactly representable as f64 (it's a power of 2)
assert_eq!(capacity, (1u128 << 80) as f64);

// No subnets allocated yet.
let allocated = datastore
.subnet_pool_allocated_count(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(allocated, 0.0);

// Allocate a /64 (2^64 addresses).
datastore
.create_external_subnet(
opctx,
&DEFAULT_SILO_ID,
&authz_project,
ExternalSubnetCreate {
identity: IdentityMetadataCreateParams {
name: "my-subnet".parse().unwrap(),
description: String::new(),
},
allocator: ExternalSubnetAllocator::Explicit {
subnet: "2001:db8:1::/64".parse().unwrap(),
},
},
)
.await
.expect("able to create external subnet");

let allocated = datastore
.subnet_pool_allocated_count(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(allocated, (1u128 << 64) as f64);

// Capacity is unchanged.
let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
assert_eq!(capacity, (1u128 << 80) as f64);

// Add a second, larger member.
let big_subnet: oxnet::IpNet = "2001:db9::/32".parse().unwrap();
datastore
.add_subnet_pool_member(
opctx,
&authz_pool,
&db_pool,
&SubnetPoolMemberAdd {
subnet: big_subnet,
min_prefix_length: Some(32),
max_prefix_length: Some(64),
},
)
.await
.expect("able to add second member");

let capacity = datastore
.subnet_pool_total_capacity(opctx, &authz_pool)
.await
.unwrap();
// /48 has 2^80, /32 has 2^96. Both are exact as f64; their sum
// is also exact because they differ by only 16 powers of 2.
assert_eq!(capacity, (1u128 << 80) as f64 + (1u128 << 96) as f64);

db.terminate().await;
logctx.cleanup_successful();
}
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

compare to

// We're breaking out the utilization tests for IPv4 and IPv6 pools, since
// pools only contain one version now.
//
// See https://github.com/oxidecomputer/omicron/issues/8888.
#[tokio::test]
async fn test_ipv4_ip_pool_utilization() {
let logctx = dev::test_setup_log("test_ipv4_ip_pool_utilization");
let db = TestDatabase::new_with_datastore(&logctx.log).await;
let (opctx, datastore) = (db.opctx(), db.datastore());
let authz_silo = opctx.authn.silo_required().unwrap();
let project = Project::new(
authz_silo.id(),
project::ProjectCreate {
identity: IdentityMetadataCreateParams {
name: "my-project".parse().unwrap(),
description: "".to_string(),
},
},
);
let (.., project) =
datastore.project_create(&opctx, project).await.unwrap();
// create an IP pool for the silo, add a range to it, and link it to the silo
let identity = IdentityMetadataCreateParams {
name: "my-pool".parse().unwrap(),
description: "".to_string(),
};
let pool = datastore
.ip_pool_create(
&opctx,
IpPool::new(
&identity,
IpVersion::V4,
IpPoolReservationType::ExternalSilos,
),
)
.await
.expect("Failed to create IP pool");
let authz_pool = authz::IpPool::new(
authz::FLEET,
pool.id(),
LookupType::ById(pool.id()),
);
// capacity of zero because there are no ranges
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 0);
let range = IpRange::V4(
Ipv4Range::new(
Ipv4Addr::new(10, 0, 0, 1),
Ipv4Addr::new(10, 0, 0, 5),
)
.unwrap(),
);
datastore
.ip_pool_add_range(&opctx, &authz_pool, &pool, &range)
.await
.expect("Could not add range");
// now it has a capacity of 5 because we added the range
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 5);
let link = IncompleteIpPoolResource {
ip_pool_id: pool.id(),
resource_type: IpPoolResourceType::Silo,
resource_id: authz_silo.id(),
is_default: true,
};
datastore
.ip_pool_link_silo(&opctx, link)
.await
.expect("Could not link pool to silo");
let ip_count = datastore
.ip_pool_allocated_count(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(ip_count, 0);
let identity = IdentityMetadataCreateParams {
name: "my-ip".parse().unwrap(),
description: "".to_string(),
};
// Allocate from default pool (no explicit pool or IP version)
let ip = datastore
.allocate_floating_ip(
&opctx,
project.id(),
identity,
FloatingIpAllocation::Auto { pool: None, ip_version: None },
)
.await
.expect("Could not allocate floating IP");
assert_eq!(ip.ip.to_string(), "10.0.0.1/32");
let ip_count = datastore
.ip_pool_allocated_count(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(ip_count, 1);
// allocating one has nothing to do with total capacity
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 5);
db.terminate().await;
logctx.cleanup_successful();
}
#[tokio::test]
async fn test_ipv6_ip_pool_utilization() {
let logctx = dev::test_setup_log("test_ipv6_ip_pool_utilization");
let db = TestDatabase::new_with_datastore(&logctx.log).await;
let (opctx, datastore) = (db.opctx(), db.datastore());
let authz_silo = opctx.authn.silo_required().unwrap();
let project = Project::new(
authz_silo.id(),
project::ProjectCreate {
identity: IdentityMetadataCreateParams {
name: "my-project".parse().unwrap(),
description: "".to_string(),
},
},
);
let (.., project) =
datastore.project_create(&opctx, project).await.unwrap();
// create an IP pool for the silo, add a range to it, and link it to the silo
let identity = IdentityMetadataCreateParams {
name: "my-pool".parse().unwrap(),
description: "".to_string(),
};
let pool = datastore
.ip_pool_create(
&opctx,
IpPool::new(
&identity,
IpVersion::V6,
IpPoolReservationType::ExternalSilos,
),
)
.await
.expect("Failed to create IP pool");
let authz_pool = authz::IpPool::new(
authz::FLEET,
pool.id(),
LookupType::ById(pool.id()),
);
let link = IncompleteIpPoolResource {
ip_pool_id: pool.id(),
resource_type: IpPoolResourceType::Silo,
resource_id: authz_silo.id(),
is_default: true,
};
datastore
.ip_pool_link_silo(&opctx, link)
.await
.expect("Could not link pool to silo");
// capacity of zero because there are no ranges
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 0);
// Add an IPv6 range
let ipv6_range = IpRange::V6(
Ipv6Range::new(
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 20),
)
.unwrap(),
);
datastore
.ip_pool_add_range(&opctx, &authz_pool, &pool, &ipv6_range)
.await
.expect("Could not add range");
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 11 + 65536);
let ip_count = datastore
.ip_pool_allocated_count(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(ip_count, 0);
let identity = IdentityMetadataCreateParams {
name: "my-ip".parse().unwrap(),
description: "".to_string(),
};
// Allocate from default pool (no explicit pool or IP version)
let ip = datastore
.allocate_floating_ip(
&opctx,
project.id(),
identity,
FloatingIpAllocation::Auto { pool: None, ip_version: None },
)
.await
.expect("Could not allocate floating IP");
assert_eq!(ip.ip.to_string(), "fd00::a/128");
let ip_count = datastore
.ip_pool_allocated_count(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(ip_count, 1);
// allocating one has nothing to do with total capacity
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 11 + 65536);
// add a giant range for fun
let ipv6_range = IpRange::V6(
Ipv6Range::new(
Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 21),
Ipv6Addr::new(
0xfd00, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
),
)
.unwrap(),
);
datastore
.ip_pool_add_range(&opctx, &authz_pool, &pool, &ipv6_range)
.await
.expect("Could not add range");
let max_ips = datastore
.ip_pool_total_capacity(&opctx, &authz_pool)
.await
.unwrap();
assert_eq!(max_ips, 1208925819614629174706166);
db.terminate().await;
logctx.cleanup_successful();
}

}
Loading
Loading