feat: Support Developer Edition connections

.gitignore

@@ -13,3 +13,4 @@
 /logs/
 .tools
 test_results.txt
+test-results

build.sh

@@ -33,12 +33,25 @@ function clean() {
 
 ## build - Builds the project without running tests.
 function build() {
-   go build -o ./cloud-sql-proxy main.go
+   local metadata="${1:-}"
+   local ldflags=""
+   if [[ -n "$metadata" ]] ; then
+       ldflags="-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata"
+   fi
+   go build -ldflags "$ldflags" -o ./cloud-sql-proxy main.go
 }
 
 ## test - Runs local unit tests.
 function test() {
-  go test -v -race -cover -short ./...
+  get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2'
+  mkdir -p test-results
+  local args=( "./..." )
+  if [[ "$#" -gt 0 ]] ; then
+    args=( "$@" )
+  fi
+  go test -v -race -cover -short "${args[@]}" -json \
+      | .tools/go-junit-report -iocopy -parser gojson -out test-results/unit.xml \
+      | jq -j 'select(.Output) | .Output '
 }
 
 ## e2e - Runs end-to-end integration tests.
@@ -53,7 +66,11 @@ function e2e() {
 # e2e_ci - Run end-to-end integration tests in the CI system.
 #   This assumes that the secrets in the env vars are already set.
 function e2e_ci() {
-  go test -race -v ./... | tee test_results.txt
+  get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2'
+  mkdir -p test-results
+  go test -race -v ./...  -json \
+     | .tools/go-junit-report -iocopy -parser gojson -out test-results/e2e.xml \
+     | jq -j 'select(.Output) | .Output '
 }
 
 function get_golang_tool() {
@@ -227,16 +244,47 @@ function write_e2e_env(){
   done
 
   # Set IAM User env vars to the local gcloud user
-  echo "export MYSQL_IAM_USER='${local_user%%@*}'"
-  echo "export POSTGRES_USER_IAM='$local_user'"
+  echo "export MYSQL_IAM_USER='$(iam_user_mysql)'"
+  echo "export POSTGRES_USER_IAM='$(iam_user_pg)'"
   } > "$1"
 
 }
 
+function iam_user_pg() {
+  # Truncate the suffix `.iam.gserviceaccount.com` if it exists. Otherwise return the email.
+  local email
+  local pguser
+
+  email="$(iam_user_email)"
+  pguser="${email%%.iam.gserviceaccount.com}"
+  if [[ -n "$pguser" ]] ; then
+    echo "$pguser"
+  else
+    echo "$email"
+  fi
+
+}
+
+function iam_user_mysql() {
+  # Truncate the part after the @
+  local email
+  local pguser
+
+  email=$(iam_user_email)
+  mysqluser="${email%%@*}"
+  echo "$mysqluser"
+}
+
+function iam_user_email() {
+  gcloud auth list --format json | jq -r '.[] | select (.status == "ACTIVE") | .account'
+}
+
+
 ## build_image - Builds and pushes the proxy container image using local source.
-## Usage: ./build.sh build_image [image-url]
+## Usage: ./build.sh build_image [image-url] [metadata]
 function build_image() {
   local image_url="${1:-}"
+  local metadata="${2:-container}"
   local push_arg=""
 
   if [[ -n "$image_url" ]]; then
@@ -254,7 +302,7 @@ function build_image() {
   trap cleanup_build EXIT
 
   echo "Building binary locally..."
-  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=container" -o cloud-sql-proxy
+  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata" -o cloud-sql-proxy
 
   echo "Creating temporary Dockerfile..."
   cat > Dockerfile.local <<EOF

cmd/root.go

@@ -584,6 +584,9 @@ the cached copy has expired. Use this setting in environments where the
 CPU may be throttled and a background refresh cannot run reliably
 (e.g., Cloud Run)`,
 	)
+	localFlags.StringVar(&c.conf.SQLDataEndpoint, "sqldata-api-endpoint", "",
+		"Override the SQL Data API endpoint",
+	)
 
 	localFlags.BoolVar(&c.conf.RunConnectionTest, "run-connection-test", false, `Runs a connection test
 against all specified instances. If an instance is unreachable, the Proxy exits with a failure
@@ -606,6 +609,10 @@ only applicable to Unix sockets)`)
 		"(*) Connect to the private ip address for all instances")
 	localFlags.BoolVar(&c.conf.PSC, "psc", false,
 		"(*) Connect to the PSC endpoint for all instances")
+	localFlags.BoolVar(&c.conf.SQLDataEnabled, "sql-data", false,
+		"Enable SQL Data to tunnel through the Cloud SQL Admin API without"+
+			" needing network access to your public or private IP",
+	)
 
 	return c
 }
@@ -898,6 +905,7 @@ and re-try with just --auto-iam-authn`)
 			p, pok := q["port"]
 			u, uok := q["unix-socket"]
 			up, upok := q["unix-socket-path"]
+			sd, sdok := q["sql-data"]
 
 			if aok && uok {
 				return newBadCommandError("cannot specify both address and unix-socket query params")
@@ -955,6 +963,16 @@ and re-try with just --auto-iam-authn`)
 				}
 				ic.UnixSocketPath = up[0]
 			}
+			if sdok {
+				if len(sd) != 1 {
+					return newBadCommandError(fmt.Sprintf("sql-data query param should be only one value %q", a))
+				}
+				if sd[0] != "true" && sd[0] != "false" {
+					return newBadCommandError(fmt.Sprintf("sql-data query param should be \"true\" or \"false\" %q", a))
+				}
+				b := sd[0] == "true"
+				ic.SQLDataEnabled = &b
+			}
 
 			ic.IAMAuthN, err = parseBoolOpt(q, "auto-iam-authn")
 			if err != nil {

cmd/root_test.go

@@ -466,6 +466,29 @@ func TestNewCommandArguments(t *testing.T) {
 				RunConnectionTest: true,
 			}),
 		},
+		{
+			desc: "using the sql-data flag",
+			args: []string{"--sql-data", "proj:region:inst"},
+			want: withDefaults(&proxy.Config{
+				SQLDataEnabled: true,
+			}),
+		},
+		{
+			desc: "using the sqldata-api-endpoint flag",
+			args: []string{"--sqldata-api-endpoint", "https://test.googleapis.com", "proj:region:inst"},
+			want: withDefaults(&proxy.Config{
+				SQLDataEndpoint: "https://test.googleapis.com",
+			}),
+		},
+		{
+			desc: "using the sql-data query param",
+			args: []string{"proj:region:inst?sql-data=true"},
+			want: withDefaults(&proxy.Config{
+				Instances: []proxy.InstanceConnConfig{{
+					SQLDataEnabled: pointer(true),
+				}},
+			}),
+		},
 	}
 
 	for _, tc := range tcs {
@@ -821,6 +844,22 @@ func TestNewCommandWithEnvironmentConfig(t *testing.T) {
 				AutoIP: true,
 			}),
 		},
+		{
+			desc:     "using the sql-data envvar",
+			envName:  "CSQL_PROXY_SQL_DATA",
+			envValue: "true",
+			want: withDefaults(&proxy.Config{
+				SQLDataEnabled: true,
+			}),
+		},
+		{
+			desc:     "using the sqldata-api-endpoint envvar",
+			envName:  "CSQL_PROXY_SQLDATA_API_ENDPOINT",
+			envValue: "https://test.googleapis.com",
+			want: withDefaults(&proxy.Config{
+				SQLDataEndpoint: "https://test.googleapis.com",
+			}),
+		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.desc, func(t *testing.T) {
@@ -1033,6 +1072,54 @@ func TestPSCQueryParams(t *testing.T) {
 	}
 }
 
+func TestSQLDataQueryParams(t *testing.T) {
+	tcs := []struct {
+		desc string
+		args []string
+		want *bool
+	}{
+		{
+			desc: "when the query string is absent",
+			args: []string{"proj:region:inst"},
+			want: nil,
+		},
+		{
+			desc: "when the query string is true",
+			args: []string{"proj:region:inst?sql-data=true"},
+			want: pointer(true),
+		},
+		{
+			desc: "when the query string is false",
+			args: []string{"proj:region:inst?sql-data=false"},
+			want: pointer(false),
+		},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.desc, func(t *testing.T) {
+			c, err := invokeProxyCommand(tc.args)
+			if err != nil {
+				t.Fatalf("command.Execute: %v", err)
+			}
+			if tc.want == nil {
+				if len(c.conf.Instances) > 0 && c.conf.Instances[0].SQLDataEnabled != nil {
+					t.Fatalf("args = %v, want nil, got = %v", tc.args, *c.conf.Instances[0].SQLDataEnabled)
+				}
+				return
+			}
+			if len(c.conf.Instances) == 0 {
+				t.Fatal("expected at least one instance")
+			}
+			got := c.conf.Instances[0].SQLDataEnabled
+			if got == nil {
+				t.Fatalf("args = %v, want = %v, got = nil", tc.args, *tc.want)
+			}
+			if *got != *tc.want {
+				t.Errorf("args = %v, want = %v, got = %v", tc.args, *tc.want, *got)
+			}
+		})
+	}
+}
+
 func TestNewCommandWithErrors(t *testing.T) {
 	tcs := []struct {
 		desc string
@@ -1152,6 +1239,14 @@ func TestNewCommandWithErrors(t *testing.T) {
 			desc: "when the iam authn login query param contains multiple values",
 			args: []string{"proj:region:inst?auto-iam-authn=true&auto-iam-authn=false"},
 		},
+		{
+			desc: "when the sql-data query param contains multiple values",
+			args: []string{"proj:region:inst?sql-data=true&sql-data=false"},
+		},
+		{
+			desc: "when the sql-data query param is bogus",
+			args: []string{"proj:region:inst?sql-data=nope"},
+		},
 		{
 			desc: "when the iam authn login query param is bogus",
 			args: []string{"proj:region:inst?auto-iam-authn=nope"},

docs/cmd/cloud-sql-proxy.md

@@ -279,7 +279,9 @@ cloud-sql-proxy INSTANCE_CONNECTION_NAME... [flags]
                                              status code.
       --skip-failed-instance-config          If set, the Proxy will skip any instances that are invalid/unreachable (
                                              only applicable to Unix sockets)
+      --sql-data                             Enable SQL Data to tunnel through the Cloud SQL Admin API without needing network access to your public or private IP
       --sqladmin-api-endpoint string         API endpoint for all Cloud SQL Admin API requests. (default: https://sqladmin.googleapis.com)
+      --sqldata-api-endpoint string          Override the SQL Data API endpoint
   -l, --structured-logs                      Enable structured logging with LogEntry format
       --telemetry-prefix string              Prefix for Cloud Monitoring metrics.
       --telemetry-project string             Enable Cloud Monitoring and Cloud Trace with the provided project ID.

internal/proxy/proxy.go

@@ -95,6 +95,8 @@
 	// necessary. If set, UnixSocketPath takes precedence over UnixSocket, Addr
 	// and Port.
 	UnixSocketPath string
+	// SQLDataEnabled enables connections through the SqlDataService for this connection.
+	SQLDataEnabled *bool
 	// IAMAuthN enables automatic IAM DB Authentication for the instance.
 	// MySQL and Postgres only. If it is nil, the value was not specified.
 	IAMAuthN *bool
@@ -200,6 +202,11 @@
 	// of a request context, e.g., Cloud Run.
 	LazyRefresh bool
 
+	// SQLDataEnabled configures the dialer to use the SQL Data API.
+	SQLDataEnabled bool
+	// SQLDataEndpoint configures the endpoint of the SQL Data service.
+	SQLDataEndpoint string
+
 	// Instances are configuration for individual instances. Instance
 	// configuration takes precedence over global configuration.
 	Instances []InstanceConnConfig
@@ -282,6 +289,9 @@
 	if i.IAMAuthN != nil {
 		opts = append(opts, cloudsqlconn.WithDialIAMAuthN(*i.IAMAuthN))
 	}
+	if i.SQLDataEnabled != nil && *i.SQLDataEnabled || c.SQLDataEnabled {
+		opts = append(opts, cloudsqlconn.WithSQLData())
+	}
 
 	switch {
 	// If private IP is enabled at the instance level, or private IP is enabled globally
@@ -469,6 +479,10 @@
 		opts = append(opts, cloudsqlconn.WithLazyRefresh())
 	}
 
+	if c.SQLDataEndpoint != "" {
+		opts = append(opts, cloudsqlconn.WithSQLDataEndpoint(c.SQLDataEndpoint))
+	}
+
 	return opts, nil
 }
 
@@ -564,8 +578,12 @@
 		return configureFUSE(c, conf)
 	}
 
+	// unless the proxy is in SqlDataEnabled mode, initiate a refresh operation to warm the cache
 	for _, inst := range conf.Instances {
-		// Initiate refresh operation and warm the cache.
+		// Skip instances with SqlDataEnabled
+		if conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled {
+			continue
+		}
 		go func(name string) { _, _ = d.EngineVersion(ctx, name) }(inst.Name)
 	}
 
@@ -859,6 +877,10 @@
 			np = inst.Port
 		case conf.Port != 0:
 			np = pc.nextPort()
+		case conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled:
+			// Only Postgres is supported by the SqlDataService
+			// when more engines are supported, this code will need to change.
+			np = pc.nextDBPort("POSTGRES")
 		default:
 			version, err := c.dialer.EngineVersion(ctx, inst.Name)
 			// Exit if the port is not specified for inactive instance
@@ -873,10 +895,17 @@
 	} else {
 		network = "unix"
 
-		version, err := c.dialer.EngineVersion(ctx, inst.Name)
-		if err != nil {
-			c.logger.Errorf("[%v] could not resolve instance version: %v", inst.Name, err)
-			return nil, err
+		var version string
+		switch {
+		case conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled:
+			version = "POSTGRES"
+		default:
+			var err error
+			version, err = c.dialer.EngineVersion(ctx, inst.Name)
+			if err != nil {
+				c.logger.Errorf("[%v] could not resolve instance version: %v", inst.Name, err)
+				return nil, err
+			}
 		}
 
 		address, err = newUnixSocketMount(inst, conf.UnixSocket, strings.HasPrefix(version, "POSTGRES"))