diff --git a/api/adc/types.go b/api/adc/types.go index 4149be1d..9a8d4f2e 100644 --- a/api/adc/types.go +++ b/api/adc/types.go @@ -806,6 +806,11 @@ type Config struct { Token string TlsVerify bool BackendType string + + // CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + // control plane, in place of the system trust store. Only meaningful when + // TlsVerify is true. + CaBundle string } // MarshalJSON implements custom JSON marshaling for adcConfig @@ -815,10 +820,12 @@ func (c Config) MarshalJSON() ([]byte, error) { Name string `json:"name"` ServerAddrs []string `json:"serverAddrs"` TlsVerify bool `json:"tlsVerify"` + HasCaBundle bool `json:"hasCaBundle"` }{ Name: c.Name, ServerAddrs: c.ServerAddrs, TlsVerify: c.TlsVerify, + HasCaBundle: c.CaBundle != "", }) } diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go index 680fa8a9..629c780c 100644 --- a/api/v1alpha1/gatewayproxy_types.go +++ b/api/v1alpha1/gatewayproxy_types.go @@ -120,6 +120,7 @@ type ControlPlaneAuth struct { // ControlPlaneProvider defines configuration for control plane provider. // +kubebuilder:validation:XValidation:rule="has(self.endpoints) != has(self.service)" // +kubebuilder:validation:XValidation:rule="oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode",message="mode is immutable" +// +kubebuilder:validation:XValidation:rule="!has(self.caBundle) || self.caBundle.contains('-----BEGIN CERTIFICATE-----')",message="caBundle must be a PEM-encoded certificate" type ControlPlaneProvider struct { // Mode specifies the mode of control plane provider. // Can be `apisix` or `apisix-standalone`. @@ -136,6 +137,13 @@ type ControlPlaneProvider struct { // +optional TlsVerify *bool `json:"tlsVerify,omitempty"` + // CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + // control plane's TLS certificate, in place of the system trust store. + // Set it when the control plane uses a self-signed or private CA certificate. + // It has no effect when tlsVerify is false. + // +optional + CaBundle string `json:"caBundle,omitempty"` + // Auth specifies the authentication configuration. // +kubebuilder:validation:Required Auth ControlPlaneAuth `json:"auth"` diff --git a/config/crd-nocel/apisix.apache.org_v2.yaml b/config/crd-nocel/apisix.apache.org_v2.yaml index 9e4cc186..bde3da1c 100644 --- a/config/crd-nocel/apisix.apache.org_v2.yaml +++ b/config/crd-nocel/apisix.apache.org_v2.yaml @@ -2273,6 +2273,13 @@ spec: required: - type type: object + caBundle: + description: |- + CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + control plane's TLS certificate, in place of the system trust store. + Set it when the control plane uses a self-signed or private CA certificate. + It has no effect when tlsVerify is false. + type: string endpoints: description: Endpoints specifies the list of control plane endpoints. diff --git a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml index 23a7ed50..617226d5 100644 --- a/config/crd/bases/apisix.apache.org_gatewayproxies.yaml +++ b/config/crd/bases/apisix.apache.org_gatewayproxies.yaml @@ -120,6 +120,13 @@ spec: - message: adminKey must be specified when type is AdminKey rule: 'self.type == ''AdminKey'' ? has(self.adminKey) : true' + caBundle: + description: |- + CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the + control plane's TLS certificate, in place of the system trust store. + Set it when the control plane uses a self-signed or private CA certificate. + It has no effect when tlsVerify is false. + type: string endpoints: description: Endpoints specifies the list of control plane endpoints. @@ -158,6 +165,9 @@ spec: - message: mode is immutable rule: oldSelf == null || (!has(self.mode) && !has(oldSelf.mode)) || self.mode == oldSelf.mode + - message: caBundle must be a PEM-encoded certificate + rule: '!has(self.caBundle) || self.caBundle.contains(''-----BEGIN + CERTIFICATE-----'')' type: description: Type specifies the type of provider. Can only be `ControlPlane`. diff --git a/docs/en/latest/reference/api-reference.md b/docs/en/latest/reference/api-reference.md index 25b729c2..742ee2e6 100644 --- a/docs/en/latest/reference/api-reference.md +++ b/docs/en/latest/reference/api-reference.md @@ -313,6 +313,7 @@ ControlPlaneProvider defines configuration for control plane provider. | `endpoints` _string array_ | Endpoints specifies the list of control plane endpoints. | | `service` _[ProviderService](#providerservice)_ | | | `tlsVerify` _boolean_ | TlsVerify specifies whether to verify the TLS certificate of the control plane. | +| `caBundle` _string_ | CaBundle is a PEM-encoded CA certificate (or bundle) used to verify the control plane's TLS certificate, in place of the system trust store. Set it when the control plane uses a self-signed or private CA certificate. It has no effect when tlsVerify is false. | | `auth` _[ControlPlaneAuth](#controlplaneauth)_ | Auth specifies the authentication configuration. | diff --git a/internal/adc/client/executor.go b/internal/adc/client/executor.go index 7223b59b..dcb232ee 100644 --- a/internal/adc/client/executor.go +++ b/internal/adc/client/executor.go @@ -79,7 +79,11 @@ type ADCServerOpts struct { LabelSelector map[string]string `json:"labelSelector,omitempty"` IncludeResourceType []string `json:"includeResourceType,omitempty"` TlsSkipVerify *bool `json:"tlsSkipVerify,omitempty"` - CacheKey string `json:"cacheKey"` + // CaCert is the PEM-encoded CA certificate (or bundle) the ADC server verifies + // the control plane against. Older ADC servers ignore it, and omitempty keeps + // requests without a CA bundle byte for byte what they were. + CaCert string `json:"caCert,omitempty"` + CacheKey string `json:"cacheKey"` } // MarshalLog implements logr.Marshaler so logging the request body redacts the @@ -93,6 +97,7 @@ func (r ADCServerRequest) MarshalLog() any { "labelSelector": r.Task.Opts.LabelSelector, "includeResourceType": r.Task.Opts.IncludeResourceType, "tlsSkipVerify": r.Task.Opts.TlsSkipVerify, + "hasCaCert": r.Task.Opts.CaCert != "", "cacheKey": r.Task.Opts.CacheKey, "config": r.Task.Config.MarshalLog(), } @@ -234,7 +239,7 @@ func (e *HTTPADCExecutor) runHTTPSyncForSingleServer(ctx context.Context, server } // Build HTTP request - req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, "/sync") + req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, "/sync") if err != nil { return fmt.Errorf("failed to build HTTP request: %w", err) } @@ -268,7 +273,7 @@ func (e *HTTPADCExecutor) runHTTPValidateForSingleServer(ctx context.Context, se return fmt.Errorf("failed to load resources from file %s: %w", filePath, err) } - req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, http.MethodPut, "/validate") + req, err := e.buildHTTPRequest(ctx, serverAddr, config, labels, types, resources, "/validate") if err != nil { return fmt.Errorf("failed to build validate request: %w", err) } @@ -339,7 +344,7 @@ func (e *HTTPADCExecutor) loadResourcesFromFile(filePath string) (*adctypes.Reso } // buildHTTPRequest builds the HTTP request for ADC Server -func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, method string, path string) (*http.Request, error) { +func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr string, config adctypes.Config, labels map[string]string, types []string, resources *adctypes.Resources, path string) (*http.Request, error) { // Prepare request body tlsVerify := config.TlsVerify reqBody := ADCServerRequest{ @@ -351,6 +356,7 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin LabelSelector: labels, IncludeResourceType: types, TlsSkipVerify: ptr.To(!tlsVerify), + CaCert: config.CaBundle, CacheKey: config.Name, }, Config: *resources, @@ -372,10 +378,11 @@ func (e *HTTPADCExecutor) buildHTTPRequest(ctx context.Context, serverAddr strin "labelSelector", labels, "includeResourceType", types, "tlsSkipVerify", !tlsVerify, + "hasCaCert", config.CaBundle != "", ) // Create HTTP request - req, err := http.NewRequestWithContext(ctx, method, e.serverURL+path, bytes.NewBuffer(jsonData)) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, e.serverURL+path, bytes.NewBuffer(jsonData)) if err != nil { return nil, fmt.Errorf("failed to create HTTP request: %w", err) } diff --git a/internal/adc/client/executor_test.go b/internal/adc/client/executor_test.go new file mode 100644 index 00000000..c52bba10 --- /dev/null +++ b/internal/adc/client/executor_test.go @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package client + +import ( + "context" + "encoding/json" + "io" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + adctypes "github.com/apache/apisix-ingress-controller/api/adc" +) + +func TestHTTPADCExecutorBuildHTTPRequestCaCert(t *testing.T) { + e := &HTTPADCExecutor{ + serverURL: "http://127.0.0.1:3000", + log: logr.Discard(), + } + + build := func(config adctypes.Config) (ADCServerOpts, string) { + req, err := e.buildHTTPRequest(context.Background(), "https://apisix:9180", config, nil, nil, + &adctypes.Resources{}, "/sync") + require.NoError(t, err) + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + var parsed ADCServerRequest + require.NoError(t, json.Unmarshal(body, &parsed)) + return parsed.Task.Opts, string(body) + } + + // Without a CA bundle the request stays what an ADC server that predates caCert + // already accepts. + opts, raw := build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true}) + assert.Empty(t, opts.CaCert) + assert.NotContains(t, raw, "caCert") + + const caCert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----" + opts, raw = build(adctypes.Config{Name: "GatewayProxy/ns/name", TlsVerify: true, CaBundle: caCert}) + assert.Equal(t, caCert, opts.CaCert) + assert.Contains(t, raw, "caCert") + // verification stays on, otherwise the bundle would be pointless + assert.Equal(t, false, *opts.TlsSkipVerify) +} diff --git a/internal/adc/translator/gatewayproxy.go b/internal/adc/translator/gatewayproxy.go index 13ace18d..aca89772 100644 --- a/internal/adc/translator/gatewayproxy.go +++ b/internal/adc/translator/gatewayproxy.go @@ -18,6 +18,8 @@ package translator import ( + "crypto/x509" + "encoding/pem" "fmt" "net" "strconv" @@ -56,6 +58,17 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte cfg.TlsVerify = *cp.TlsVerify } + if cp.CaBundle != "" { + // reject unusable CA material here rather than at connect time + if err := validateCaBundle(cp.CaBundle); err != nil { + return nil, err + } + if !cfg.TlsVerify { + t.Log.Info("caBundle is ignored because tlsVerify is disabled", "gatewayproxy", utils.NamespacedNameKind(gatewayProxy)) + } + cfg.CaBundle = cp.CaBundle + } + if cp.Auth.Type == v1alpha1.AuthTypeAdminKey && cp.Auth.AdminKey != nil { if cp.Auth.AdminKey.ValueFrom != nil && cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef @@ -142,3 +155,28 @@ func (t *Translator) TranslateGatewayProxyToConfig(tctx *provider.TranslateConte return &cfg, nil } + +// validateCaBundle parses every certificate in the bundle. x509.CertPool skips +// blocks it cannot decode, so a bundle whose second certificate is broken would +// otherwise reach the ADC server and fail there instead. +func validateCaBundle(caBundle string) error { + var count int + for rest := []byte(caBundle); len(rest) > 0; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + return fmt.Errorf("invalid caBundle: expected a CERTIFICATE block, got %s", block.Type) + } + if _, err := x509.ParseCertificate(block.Bytes); err != nil { + return fmt.Errorf("invalid caBundle: %w", err) + } + count++ + } + if count == 0 { + return errors.New("invalid caBundle: no PEM-encoded certificate found") + } + return nil +} diff --git a/internal/adc/translator/gatewayproxy_test.go b/internal/adc/translator/gatewayproxy_test.go new file mode 100644 index 00000000..3429b711 --- /dev/null +++ b/internal/adc/translator/gatewayproxy_test.go @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/apache/apisix-ingress-controller/api/v1alpha1" + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +func newGatewayProxy(tlsVerify *bool, caBundle string) *v1alpha1.GatewayProxy { + return &v1alpha1.GatewayProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gp", + }, + Spec: v1alpha1.GatewayProxySpec{ + Provider: &v1alpha1.GatewayProxyProvider{ + Type: v1alpha1.ProviderTypeControlPlane, + ControlPlane: &v1alpha1.ControlPlaneProvider{ + Endpoints: []string{"https://cp.example.com:9180"}, + TlsVerify: tlsVerify, + CaBundle: caBundle, + Auth: v1alpha1.ControlPlaneAuth{ + Type: v1alpha1.AuthTypeAdminKey, + AdminKey: &v1alpha1.AdminKeyAuth{ + Value: "admin-key", + }, + }, + }, + }, + }, + } +} + +func TestTranslateGatewayProxyToConfigCaBundle(t *testing.T) { + t.Run("carries the CA bundle into the config", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), testCACert), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.True(t, cfg.TlsVerify) + assert.Equal(t, testCACert, cfg.CaBundle) + }) + + t.Run("leaves the CA bundle empty when unset", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), ""), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Empty(t, cfg.CaBundle) + }) + + // every certificate is parsed: x509.CertPool silently skips the blocks it + // cannot decode, which would let a broken one through to the ADC server. + for name, caBundle := range map[string]string{ + "not PEM at all": "not-a-certificate", + "a header with no certificate": "-----BEGIN CERTIFICATE-----", + "an unparseable body": "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----", + "a key rather than a certificate": "-----BEGIN RSA PRIVATE KEY-----\nAAAA\n-----END RSA PRIVATE KEY-----", + "one good and one broken certificate": testCACert + + "\n-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----", + } { + t.Run("rejects a CA bundle that is "+name, func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), caBundle), false) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid caBundle") + assert.Nil(t, cfg) + }) + } + + t.Run("accepts a bundle of several certificates", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + bundle := testCACert + "\n" + testCACert + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(true), bundle), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Equal(t, bundle, cfg.CaBundle) + }) + + t.Run("still carries the CA bundle when verification is off", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + tctx := provider.NewDefaultTranslateContext(context.Background()) + + cfg, err := tr.TranslateGatewayProxyToConfig(tctx, newGatewayProxy(ptr.To(false), testCACert), false) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.False(t, cfg.TlsVerify) + assert.Equal(t, testCACert, cfg.CaBundle) + }) +}