RFC0055 Identity-Aware Routing

assets/proxy/main.go

@@ -24,6 +24,8 @@ func main() {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/proxy/", proxyHandler)
 	mux.HandleFunc("/https_proxy/", httpsProxyHandler)
+	mux.HandleFunc("/mtls_proxy/", mtlsProxyHandler)
+	mux.HandleFunc("/headers", headersHandler)
 	mux.HandleFunc("/", infoHandler(systemPort))
 
 	server := &http.Server{
@@ -91,6 +93,93 @@ func handleRequest(destination string, resp http.ResponseWriter, req *http.Reque
 	_, _ = resp.Write(readBytes)
 }
 
+func headersHandler(resp http.ResponseWriter, req *http.Request) {
+	headers := make(map[string]string)
+	for name, values := range req.Header {
+		headers[name] = strings.Join(values, ", ")
+	}
+	resp.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(resp).Encode(headers)
+}
+
+func mtlsProxyHandler(resp http.ResponseWriter, req *http.Request) {
+	destination := strings.TrimPrefix(req.URL.Path, "/mtls_proxy/")
+	destination = fmt.Sprintf("https://%s", destination)
+
+	certFile := os.Getenv("CF_INSTANCE_CERT")
+	keyFile := os.Getenv("CF_INSTANCE_KEY")
+	if certFile == "" || keyFile == "" {
+		resp.WriteHeader(http.StatusInternalServerError)
+		json.NewEncoder(resp).Encode(map[string]interface{}{
+			"status":      "error",
+			"status_code": 500,
+			"error":       "CF_INSTANCE_CERT or CF_INSTANCE_KEY not set",
+		})
+		return
+	}
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		resp.WriteHeader(http.StatusInternalServerError)
+		json.NewEncoder(resp).Encode(map[string]interface{}{
+			"status":      "error",
+			"status_code": 500,
+			"error":       fmt.Sprintf("failed to load client cert: %s", err),
+		})
+		return
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			Dial: (&net.Dialer{
+				Timeout:   10 * time.Second,
+				KeepAlive: 0,
+			}).Dial,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+				Certificates:      []tls.Certificate{cert},
+			},
+		},
+	}
+
+	getResp, err := client.Get(destination)
+	if err != nil {
+		resp.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(resp).Encode(map[string]interface{}{
+			"status":      "error",
+			"status_code": 0,
+			"error":       fmt.Sprintf("request failed: %s", err),
+		})
+		return
+	}
+	defer getResp.Body.Close()
+
+	body, err := io.ReadAll(getResp.Body)
+	if err != nil {
+		resp.WriteHeader(http.StatusInternalServerError)
+		json.NewEncoder(resp).Encode(map[string]interface{}{
+			"status":      "error",
+			"status_code": 0,
+			"error":       fmt.Sprintf("read body failed: %s", err),
+		})
+		return
+	}
+
+	respHeaders := make(map[string]string)
+	for name, values := range getResp.Header {
+		respHeaders[name] = strings.Join(values, ", ")
+	}
+
+	resp.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(resp).Encode(map[string]interface{}{
+		"status":      "success",
+		"status_code": getResp.StatusCode,
+		"body":        string(body),
+		"headers":     respHeaders,
+	})
+}
+
 var httpClient = &http.Client{
 	Transport: &http.Transport{
 		DisableKeepAlives: true,

cats_suite_helpers/cats_suite_helpers.go

@@ -250,6 +250,17 @@ func CommaDelimitedSecurityGroupsDescribe(description string, callback func()) b
 	})
 }
 
+func IdentityAwareRoutingDescribe(description string, callback func()) bool {
+	return Describe("[identity aware routing]", func() {
+		BeforeEach(func() {
+			if !Config.GetIncludeIdentityAwareRouting() {
+				Skip(skip_messages.SkipIdentityAwareRoutingMessage)
+			}
+		})
+		Describe(description, callback)
+	})
+}
+
 func ServiceDiscoveryDescribe(description string, callback func()) bool {
 	return Describe("[service discovery]", func() {
 		BeforeEach(func() {

cats_suite_test.go

@@ -20,6 +20,7 @@ import (
 	_ "github.com/cloudfoundry/cf-acceptance-tests/docker"
 	_ "github.com/cloudfoundry/cf-acceptance-tests/file_based_service_bindings"
 	_ "github.com/cloudfoundry/cf-acceptance-tests/http2_routing"
+	_ "github.com/cloudfoundry/cf-acceptance-tests/identity_aware_routing"
 	_ "github.com/cloudfoundry/cf-acceptance-tests/internet_dependent"
 	_ "github.com/cloudfoundry/cf-acceptance-tests/ipv6"
 	_ "github.com/cloudfoundry/cf-acceptance-tests/isolation_segments"

helpers/config/config.go

@@ -24,6 +24,8 @@ type CatsConfig interface {
 	GetIncludeSecurityGroups() bool
 	GetIncludeServices() bool
 	GetIncludeUserProvidedServices() bool
+	GetIncludeIdentityAwareRouting() bool
+	GetIdentityAwareDomain() string
 	GetIncludeServiceDiscovery() bool
 	GetIncludeSsh() bool
 	GetIncludeTasks() bool

helpers/config/config_struct.go

@@ -90,6 +90,8 @@ type config struct {
 	IncludeRoutingIsolationSegments *bool `json:"include_routing_isolation_segments"`
 	IncludeSSO                      *bool `json:"include_sso"`
 	IncludeSecurityGroups           *bool `json:"include_security_groups"`
+	IncludeIdentityAwareRouting     *bool   `json:"include_identity_aware_routing"`
+	IdentityAwareDomain            *string `json:"identity_aware_domain"`
 	IncludeServiceDiscovery         *bool `json:"include_service_discovery"`
 	IncludeServiceInstanceSharing   *bool `json:"include_service_instance_sharing"`
 	IncludeServices                 *bool `json:"include_services"`
@@ -199,6 +201,8 @@ func getDefaults() config {
 	defaults.IncludeRouteServices = ptrToBool(false)
 	defaults.IncludeSSO = ptrToBool(false)
 	defaults.IncludeSecurityGroups = ptrToBool(false)
+	defaults.IncludeIdentityAwareRouting = ptrToBool(false)
+	defaults.IdentityAwareDomain = ptrToString("apps.identity")
 	defaults.IncludeServiceDiscovery = ptrToBool(false)
 	defaults.IncludeServices = ptrToBool(false)
 	defaults.IncludeUserProvidedServices = ptrToBool(false)
@@ -479,6 +483,9 @@ func validateConfig(config *config) error {
 	if config.IncludeSecurityGroups == nil {
 		errs = errors.Join(errs, fmt.Errorf("* 'include_security_groups' must not be null"))
 	}
+	if config.IncludeIdentityAwareRouting == nil {
+		errs = errors.Join(errs, fmt.Errorf("* 'include_identity_aware_routing' must not be null"))
+	}
 	if config.IncludeServiceDiscovery == nil {
 		errs = errors.Join(errs, fmt.Errorf("* 'include_service_discovery' must not be null"))
 	}
@@ -1115,6 +1122,14 @@ func (c *config) GetIncludeWindows() bool {
 	return *c.IncludeWindows
 }
 
+func (c *config) GetIncludeIdentityAwareRouting() bool {
+	return *c.IncludeIdentityAwareRouting
+}
+
+func (c *config) GetIdentityAwareDomain() string {
+	return *c.IdentityAwareDomain
+}
+
 func (c *config) GetIncludeServiceDiscovery() bool {
 	return *c.IncludeServiceDiscovery
 }

helpers/skip_messages/skip_messages.go

@@ -55,6 +55,8 @@ const SkipIsolationSegmentsMessage = `Skipping this test because config.IncludeI
 const SkipRoutingIsolationSegmentsMessage = `Skipping this test because Config.IncludeRoutingIsolationSegments is set to 'false'.`
 const SkipZipkinMessage = `Skipping this test because config.IncludeZipkin is set to 'false'`
 const SkipServiceDiscoveryMessage = `Skipping this test because config.IncludeServiceDiscovery is set to 'false'.`
+const SkipIdentityAwareRoutingMessage = `Skipping this test because config.IncludeIdentityAwareRouting is set to 'false'.
+NOTE: Ensure that identity-aware routing is enabled and the identity-aware domain is configured before running this test.`
 const SkipServiceInstanceSharingMessage = `Skipping this test because config.IncludeServiceInstanceSharing is set to 'false'.`
 const SkipCapiExperimentalMessage = `Skipping this test because config.IncludeCapiExperimental is set to 'false'.`
 const SkipWindowsTasksMessage = `Skipping Windows tasks tests (requires diego-release v1.20.0 and above)`

identity_aware_routing/identity_aware_routing.go

@@ -0,0 +1,183 @@
+package identity_aware_routing
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
+	"github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gexec"
+
+	"github.com/cloudfoundry/cf-test-helpers/v2/cf"
+	"github.com/cloudfoundry/cf-test-helpers/v2/helpers"
+)
+
+type mtlsProxyResponse struct {
+	Status     string            `json:"status"`
+	StatusCode int               `json:"status_code"`
+	Body       string            `json:"body"`
+	Headers    map[string]string `json:"headers"`
+	Error      string            `json:"error"`
+}
+
+var _ = IdentityAwareRoutingDescribe("Identity-Aware Routing", func() {
+	var appNameFrontend string
+	var appNameBackend string
+	var appNameUnauthorized string
+	var backendHostName string
+	var identityAwareDomain string
+
+	BeforeEach(func() {
+		identityAwareDomain = Config.GetIdentityAwareDomain()
+
+		backendHostName = random_name.CATSRandomName("HOST")
+		appNameFrontend = random_name.CATSRandomName("APP-FRONT")
+		appNameBackend = random_name.CATSRandomName("APP-BACK")
+		appNameUnauthorized = random_name.CATSRandomName("APP-UNAUTH")
+
+		// push backend app (proxy app so it has /headers endpoint)
+		Expect(cf.Cf(
+			"push", appNameBackend,
+			"-b", Config.GetGoBuildpackName(),
+			"-m", DEFAULT_MEMORY_LIMIT,
+			"-p", assets.NewAssets().Proxy,
+			"-f", assets.NewAssets().Proxy+"/manifest.yml",
+		).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
+
+		// map identity-aware route to backend app
+		Expect(cf.Cf("map-route", appNameBackend, identityAwareDomain, "--hostname", backendHostName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
+
+		// push frontend app (proxy app with /mtls_proxy endpoint)
+		Expect(cf.Cf(
+			"push", appNameFrontend,
+			"-b", Config.GetGoBuildpackName(),
+			"-m", DEFAULT_MEMORY_LIMIT,
+			"-p", assets.NewAssets().Proxy,
+			"-f", assets.NewAssets().Proxy+"/manifest.yml",
+		).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
+
+		// push unauthorized app (same proxy app, different identity)
+		Expect(cf.Cf(
+			"push", appNameUnauthorized,
+			"-b", Config.GetGoBuildpackName(),
+			"-m", DEFAULT_MEMORY_LIMIT,
+			"-p", assets.NewAssets().Proxy,
+			"-f", assets.NewAssets().Proxy+"/manifest.yml",
+		).Wait(Config.CfPushTimeoutDuration())).To(Exit(0))
+	})
+
+	AfterEach(func() {
+		app_helpers.AppReport(appNameFrontend)
+		app_helpers.AppReport(appNameBackend)
+		app_helpers.AppReport(appNameUnauthorized)
+
+		Expect(cf.Cf("delete", appNameFrontend, "-f", "-r").Wait()).To(Exit(0))
+		Expect(cf.Cf("delete", appNameBackend, "-f", "-r").Wait()).To(Exit(0))
+		Expect(cf.Cf("delete", appNameUnauthorized, "-f", "-r").Wait()).To(Exit(0))
+	})
+
+	mtlsProxyURL := func(appName, backendHost, domain, path string) string {
+		return fmt.Sprintf("%s%s.%s/mtls_proxy/%s.%s/%s",
+			Config.Protocol(), appName, Config.GetAppsDomain(),
+			backendHost, domain, path)
+	}
+
+	curlMtlsProxy := func(appName, backendHost, domain, path string) mtlsProxyResponse {
+		curlArgs := mtlsProxyURL(appName, backendHost, domain, path)
+		curl := helpers.Curl(Config, curlArgs).Wait()
+		var resp mtlsProxyResponse
+		err := json.Unmarshal(curl.Out.Contents(), &resp)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to parse mtls_proxy response: %s", string(curl.Out.Contents()))
+		return resp
+	}
+
+	Describe("mTLS authorization with access rules", func() {
+	It("denies access by default and allows after adding an access rule", func() {
+		By("verifying the frontend is denied without access rules (default deny)")
+		Eventually(func() int {
+			resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers")
+			return resp.StatusCode
+		}, 2*time.Minute).Should(Equal(403))
+
+		By("creating an access rule for the frontend app")
+		Expect(cf.Cf(
+			"add-access-rule", identityAwareDomain,
+			"--source-app", appNameFrontend,
+			"--hostname", backendHostName,
+		).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
+
+		By("verifying the access rule is listed")
+		accessRulesOutput := cf.Cf("access-rules", "--domain", identityAwareDomain).Wait(Config.DefaultTimeoutDuration())
+		Expect(accessRulesOutput).To(Exit(0))
+		Expect(string(accessRulesOutput.Out.Contents())).To(ContainSubstring(appNameFrontend))
+
+		By("verifying the frontend can now reach the backend")
+		Eventually(func() int {
+			resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers")
+			return resp.StatusCode
+		}, 2*time.Minute).Should(Equal(200))
+	})
+
+	It("denies access from an unauthorized app even with a valid certificate", func() {
+		By("creating an access rule only for the frontend app")
+		Expect(cf.Cf(
+			"add-access-rule", identityAwareDomain,
+			"--source-app", appNameFrontend,
+			"--hostname", backendHostName,
+		).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
+
+		By("verifying the authorized frontend can reach the backend")
+		Eventually(func() int {
+			resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers")
+			return resp.StatusCode
+		}, 2*time.Minute).Should(Equal(200))
+
+		By("verifying the unauthorized app is denied")
+		Consistently(func() int {
+			resp := curlMtlsProxy(appNameUnauthorized, backendHostName, identityAwareDomain, "headers")
+			return resp.StatusCode
+		}, 30*time.Second).Should(Equal(403))
+	})
+
+		It("forwards X-Forwarded-Client-Cert header with caller identity in Envoy format", func() {
+			frontendGuid := GuidForAppName(appNameFrontend)
+
+			By("creating an access rule for the frontend app")
+			Expect(cf.Cf(
+				"add-access-rule", identityAwareDomain,
+				"--source-app", appNameFrontend,
+				"--hostname", backendHostName,
+			).Wait(Config.DefaultTimeoutDuration())).To(Exit(0))
+
+		By("calling the backend and examining the XFCC header")
+		var xfcc string
+		Eventually(func() string {
+			resp := curlMtlsProxy(appNameFrontend, backendHostName, identityAwareDomain, "headers")
+			if resp.StatusCode != 200 {
+				return ""
+			}
+			// The backend returns its request headers as JSON via /headers
+			var headers map[string]string
+			err := json.Unmarshal([]byte(resp.Body), &headers)
+			if err != nil {
+				return ""
+			}
+			xfcc = headers["X-Forwarded-Client-Cert"]
+			return xfcc
+		}, 2*time.Minute).ShouldNot(BeEmpty())
+
+			By("verifying the XFCC header is in Envoy format")
+			Expect(xfcc).To(ContainSubstring("Hash="))
+			Expect(xfcc).To(ContainSubstring("Subject="))
+
+			By("verifying the XFCC header contains the frontend app GUID")
+			Expect(strings.ToLower(xfcc)).To(ContainSubstring("ou=app:" + strings.ToLower(frontendGuid)))
+		})
+	})
+})