feat: give every instance its own network interface

api/v1alpha/groupversion_info.go

@@ -4,17 +4,37 @@
 package v1alpha
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/controller-runtime/pkg/scheme"
 )
 
 var (
 	// GroupVersion is group version used to register these objects
 	GroupVersion = schema.GroupVersion{Group: "compute.datumapis.com", Version: "v1alpha"}
 
 	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
-	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+	SchemeBuilder = &objectSchemeBuilder{}
 
 	// AddToScheme adds the types in this group-version to the given scheme.
 	AddToScheme = SchemeBuilder.AddToScheme
 )
+
+// objectSchemeBuilder registers API objects against GroupVersion. API packages
+// must stay cheap to import, so this mirrors what the deprecated
+// controller-runtime scheme.Builder did without depending on controller-runtime.
+//
+// +kubebuilder:object:generate=false
+type objectSchemeBuilder struct {
+	runtime.SchemeBuilder
+}
+
+// Register adds one or more objects to the builder so they can be added to a scheme.
+func (b *objectSchemeBuilder) Register(objects ...runtime.Object) *objectSchemeBuilder {
+	b.SchemeBuilder.Register(func(s *runtime.Scheme) error {
+		s.AddKnownTypes(GroupVersion, objects...)
+		metav1.AddToGroupVersion(s, GroupVersion)
+		return nil
+	})
+	return b
+}

api/v1alpha/instance_types.go

@@ -16,8 +16,17 @@ type InstanceSpec struct {
 
 	// Network interface configuration.
 	//
+	// Keyed by interface name so an interface keeps its identity, and therefore
+	// its addresses, across updates to the rest of the list.
+	//
+	// Limited to a single interface until the data plane can attach more than
+	// one to an instance.
+	//
 	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=1
+	// +listType=map
+	// +listMapKey=name
 	NetworkInterfaces []InstanceNetworkInterface `json:"networkInterfaces,omitempty"`
 
 	// Volumes that must be available to attach to an instance's containers or
@@ -295,12 +304,84 @@ type InstanceRuntimeResources struct {
 	Requests corev1.ResourceList `json:"requests,omitempty"`
 }
 
+// InstanceNetworkInterface describes one interface an instance needs. The
+// fields beyond `network` and `networkPolicy` are copied verbatim onto the
+// NetworkInterfaceClaim created for each instance slot, so they carry the same
+// meaning, defaults, and immutability the claim API defines.
+//
+// The location an interface is claimed in is implicit: the claim is created in
+// the control plane serving the instance, which is already location scoped.
+//
+// +kubebuilder:validation:XValidation:message="addresses is immutable and cannot be set, changed, or cleared after creation",rule="has(self.addresses) == has(oldSelf.addresses) && (!has(self.addresses) || self.addresses == oldSelf.addresses)"
 type InstanceNetworkInterface struct {
 	// The network to attach the network interface to.
 	//
 	// +kubebuilder:validation:Required
 	Network networkingv1alpha.NetworkRef `json:"network"`
 
+	// The name of the interface, such as eth0 or eth1. It is both the device
+	// name the guest operating system sees and the suffix of the interface
+	// claim's name, which is what keeps an interface's addresses with the
+	// instance slot across replacement.
+	//
+	// Immutable, because the guest is configured against it and the claim is
+	// named after it.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=15
+	// +kubebuilder:default="eth0"
+	// +kubebuilder:validation:XValidation:message="name is immutable and cannot be changed after creation",rule="self == oldSelf"
+	Name string `json:"name,omitempty"`
+
+	// The address families the interface must carry, in priority order. List
+	// [IPv6, IPv4] for a dual-stack interface. The first family listed holds the
+	// interface's primary address, which is the one reported as the instance's
+	// network IP.
+	//
+	// Every family listed must be satisfiable or the interface is never
+	// published, so asking for a family the network does not carry fails rather
+	// than yielding a partially addressed interface.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=2
+	// +kubebuilder:default={IPv6}
+	// +kubebuilder:validation:XValidation:message="Each address family may be requested at most once",rule="self.all(f, self.exists_one(g, g == f))"
+	// +kubebuilder:validation:XValidation:message="ipFamilies is immutable and cannot be changed after creation",rule="self == oldSelf"
+	IPFamilies []networkingv1alpha.IPFamily `json:"ipFamilies,omitempty"`
+
+	// Requests for addresses beyond the ones the interface holds inside its
+	// network, such as a public IPv4 address in front of a private one. Each is
+	// reported in the interface's `externalAddresses` status.
+	//
+	// Omit this field for ordinary private addressing, which is the common case.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=4
+	// +kubebuilder:validation:XValidation:message="Each address class may be requested at most once",rule="self.all(a, self.exists_one(b, b.class == a.class))"
+	Addresses []InstanceNetworkInterfaceAddressRequest `json:"addresses,omitempty"`
+
+	// What becomes of the interface, and its addresses, when the instance slot
+	// it serves goes away.
+	//
+	// Delete returns the addresses to IPAM, so an instance recreated later comes
+	// back on different addresses. Retain keeps them reserved, and billable, so a
+	// later instance filling the same slot returns to the same addresses. Choose
+	// Retain when an address is published in DNS, allowed through a firewall, or
+	// otherwise depended on from outside.
+	//
+	// Both policies keep the addresses for as long as the slot exists, including
+	// across instance replacement. They differ only on scale-down and deletion.
+	//
+	// Immutable. An address keeps the policy it was allocated under.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default="Delete"
+	// +kubebuilder:validation:XValidation:message="reclaimPolicy is immutable and cannot be changed after creation",rule="self == oldSelf"
+	ReclaimPolicy networkingv1alpha.NetworkInterfaceReclaimPolicy `json:"reclaimPolicy,omitempty"`
+
 	// Interface specific network policy.
 	//
 	// If provided, this will result in a platform managed network policy being
@@ -312,16 +393,136 @@ type InstanceNetworkInterface struct {
 	NetworkPolicy *InstanceNetworkInterfaceNetworkPolicy `json:"networkPolicy,omitempty"`
 }
 
+// InstanceNetworkInterfaceAddressRequest asks for one address beyond the ones
+// the interface holds inside its network.
+type InstanceNetworkInterfaceAddressRequest struct {
+	// The IPAM class to allocate from, such as public-ipv4.
+	//
+	// A class names a kind of address, and the platform decides which pool and
+	// prefix length serve it. A class never names a pool, a prefix length, or a
+	// CIDR, so a class cannot be used to ask for a particular address.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=63
+	Class string `json:"class"`
+}
+
 type InstanceNetworkInterfaceStatus struct {
+	// The name of the interface this entry reports on, matching the name in the
+	// instance's spec.
+	//
+	// +kubebuilder:validation:Optional
+	Name string `json:"name,omitempty"`
+
+	// The addresses the interface holds inside its network, each with its prefix
+	// length and, once the location has a subnet, its gateway.
+	//
+	// +kubebuilder:validation:Optional
+	Addresses []InstanceNetworkInterfaceAddress `json:"addresses,omitempty"`
+
+	// The addresses the interface is reachable at from outside its network, one
+	// per class requested in the spec. Each is a bare address with no prefix
+	// length.
+	//
+	// +kubebuilder:validation:Optional
+	ExternalAddresses []InstanceNetworkInterfaceExternalAddress `json:"externalAddresses,omitempty"`
+
+	// The observations of this interface's current state. Known condition types
+	// are "Allocated" and "Programmed".
+	//
+	// +kubebuilder:validation:Optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// Single address projections of the fields above, kept for clients that read
+	// one address per interface.
+	//
+	// +kubebuilder:validation:Optional
 	Assignments InstanceNetworkInterfaceAssignmentsStatus `json:"assignments,omitempty"`
 }
 
+// InstanceNetworkInterfaceAddress is an address the interface holds inside its
+// network. These are configured on the NIC itself, and always carry a prefix
+// length.
+type InstanceNetworkInterfaceAddress struct {
+	// The address family of this entry.
+	//
+	// +kubebuilder:validation:Required
+	Family networkingv1alpha.IPFamily `json:"family"`
+
+	// The address the interface holds, in CIDR notation, such as 10.128.0.2/32
+	// or 2001:db8:a001::1/128.
+	//
+	// For IPv6 this may be a block delegated to the interface rather than a
+	// single address, such as 2001:db8:a001::/96. The interface owns the whole
+	// block and assigns within it.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=45
+	Address string `json:"address"`
+
+	// The next hop the interface routes through for this family, such as
+	// 10.128.0.1. It is empty until the subnet backing the network in this
+	// location exists.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MaxLength=45
+	Gateway string `json:"gateway,omitempty"`
+
+	// Marks the address projected into `assignments.networkIP`.
+	//
+	// Exactly one address is primary for the interface as a whole, not one per
+	// family. It is the address of the first family listed in `ipFamilies`.
+	//
+	// +kubebuilder:validation:Optional
+	Primary bool `json:"primary,omitempty"`
+
+	// The IPAM class this address was allocated from, such as private-ipv6. It
+	// is empty for addresses requested by family rather than by class.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MaxLength=63
+	Class string `json:"class,omitempty"`
+}
+
+// InstanceNetworkInterfaceExternalAddress is an address reachable from outside
+// the network, mapped onto an address the interface holds inside it. A public
+// IPv4 address in front of a private address is the usual case.
+//
+// Unlike an interface address, it is a bare address with no prefix length,
+// because nothing configures it on the NIC.
+type InstanceNetworkInterfaceExternalAddress struct {
+	// The address family of this entry.
+	//
+	// +kubebuilder:validation:Required
+	Family networkingv1alpha.IPFamily `json:"family"`
+
+	// The externally reachable address, such as 203.0.113.10. It carries no
+	// prefix length.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=45
+	Address string `json:"address"`
+
+	// The IPAM class this address was allocated from, such as public-ipv4. It
+	// matches a class requested in the interface's `addresses`.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=63
+	Class string `json:"class"`
+}
+
 type InstanceNetworkInterfaceAssignmentsStatus struct {
-	// The IP address assigned as the primary IP from the attached network.
+	// The IP address assigned as the primary IP from the attached network. It is
+	// a projection of the primary entry in the interface's `addresses`.
 	NetworkIP *string `json:"networkIP,omitempty"`
 
 	// The external IP address used for the interface. A one to one NAT will be
-	// performed for this address with the interface's network IP.
+	// performed for this address with the interface's network IP. It is a
+	// projection of the first entry in the interface's `externalAddresses`.
 	ExternalIP *string `json:"externalIP,omitempty"`
 }
 
@@ -487,6 +688,20 @@ const (
 	ReferencedDataReady = "ReferencedDataReady"
 )
 
+// Condition types reported per network interface in
+// InstanceNetworkInterfaceStatus.Conditions. They mirror the conditions the
+// networking API reports on an interface claim, so a client reads the
+// instance's interface rather than following the reference.
+const (
+	// InstanceNetworkInterfaceAllocated indicates that every requested address
+	// family, and every requested class, holds an address.
+	InstanceNetworkInterfaceAllocated = "Allocated"
+
+	// InstanceNetworkInterfaceProgrammed indicates that the data plane carries
+	// the interface's addresses. Traffic flows only once this is true.
+	InstanceNetworkInterfaceProgrammed = "Programmed"
+)
+
 const (
 	// ReferencedDataReasonResolving indicates the resolver is in the process of
 	// reading source ConfigMaps/Secrets from the project control plane.