feat(gke): refactor nginx benchmark to 3-tier architecture (client-proxy-upstream)

jimmycgz · jimmycgz · commit 36298bcd8759 · 2026-02-10T14:52:40.000-05:00
diff --git a/perfkitbenchmarker/data/container/kubernetes_nginx/nginx_proxy.yaml.j2 b/perfkitbenchmarker/data/container/kubernetes_nginx/nginx_proxy.yaml.j2
@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-proxy-deployment
+spec:
+  replicas: {{ nginx_proxy_replicas }}
+  selector:
+    matchLabels:
+      app: nginx-proxy
+  template:
+    metadata:
+      labels:
+        app: nginx-proxy
+    spec:
+      {% if runtime_class_name %}
+      runtimeClassName: {{ runtime_class_name }}
+      {% endif %}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: pkb_nodepool
+                operator: In
+                values:
+                - nginx
+      initContainers:
+      - name: setup-ssl
+        image: {{ nginx_image }}
+        command: ['/bin/bash', '-c']
+        args:
+        - |
+          mkdir -p /etc/nginx/ssl
+          openssl req -x509 -nodes -newkey ec:<(openssl ecparam -name secp384r1) -keyout /etc/nginx/ssl/ecdsa.key -out /etc/nginx/ssl/ecdsa.crt -days 365 -subj "/CN=$HOSTNAME"
+        volumeMounts:
+        - name: ssl-volume
+          mountPath: /etc/nginx/ssl
+      containers:
+      - name: nginx-proxy
+        image: {{ nginx_image }}
+        ports:
+        - containerPort: {{ nginx_port }}
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/nginx/nginx.conf
+          subPath: nginx-proxy.conf
+        - name: ssl-volume
+          mountPath: /etc/nginx/ssl
+      volumes:
+      - name: ssl-volume
+        emptyDir: {}
+      - name: config-volume
+        configMap:
+          name: nginx-configs
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-cluster
+spec:
+  type: LoadBalancer
+  ports:
+  - port: {{ nginx_port }}
+    targetPort: {{ nginx_port }}
+  selector:
+    app: nginx-proxy
diff --git a/perfkitbenchmarker/data/container/kubernetes_nginx/nginx_upstream.yaml.j2 b/perfkitbenchmarker/data/container/kubernetes_nginx/nginx_upstream.yaml.j2
@@ -0,0 +1,79 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-upstream-deployment
+spec:
+  replicas: {{ nginx_upstream_replicas }}
+  selector:
+    matchLabels:
+      app: nginx-upstream
+  template:
+    metadata:
+      labels:
+        app: nginx-upstream
+    spec:
+      {% if runtime_class_name %}
+      runtimeClassName: {{ runtime_class_name }}
+      {% endif %}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: pkb_nodepool
+                operator: In
+                values:
+                - upstream
+      initContainers:
+      - name: setup-random-content
+        image: busybox:1.35
+        command: ['sh', '-c']
+        args:
+        - |
+          dd bs=1 count={{ nginx_content_size }} if=/dev/urandom of=/usr/share/nginx/html/random_content
+        volumeMounts:
+        - name: html-volume
+          mountPath: /usr/share/nginx/html
+      - name: setup-ssl
+        image: {{ nginx_image }}
+        command: ['/bin/bash', '-c']
+        args:
+        - |
+          mkdir -p /etc/nginx/ssl
+          openssl req -x509 -nodes -newkey ec:<(openssl ecparam -name secp384r1) -keyout /etc/nginx/ssl/ecdsa.key -out /etc/nginx/ssl/ecdsa.crt -days 365 -subj "/CN=$HOSTNAME"
+        volumeMounts:
+        - name: ssl-volume
+          mountPath: /etc/nginx/ssl
+      containers:
+      - name: nginx-upstream
+        image: {{ nginx_image }}
+        ports:
+        - containerPort: {{ nginx_port }}
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/nginx/nginx.conf
+          subPath: nginx-upstream.conf
+        - name: html-volume
+          mountPath: /usr/share/nginx/html
+        - name: ssl-volume
+          mountPath: /etc/nginx/ssl
+      volumes:
+      - name: ssl-volume
+        emptyDir: {}
+      - name: config-volume
+        configMap:
+          name: nginx-configs
+      - name: html-volume
+        emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-upstream
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ nginx_port }}
+    targetPort: {{ nginx_port }}
+  selector:
+    app: nginx-upstream
diff --git a/perfkitbenchmarker/linux_benchmarks/kubernetes_nginx_benchmark.py b/perfkitbenchmarker/linux_benchmarks/kubernetes_nginx_benchmark.py
@@ -14,19 +14,27 @@
 """Runs wrk2 clients against replicated nginx servers behind a load balancer."""
 
 import functools
+import logging
 import os
 import shutil
 import tempfile
+import time
 
 from absl import flags
 from perfkitbenchmarker import background_tasks
 from perfkitbenchmarker import configs
 from perfkitbenchmarker import data
+from perfkitbenchmarker import errors
 from perfkitbenchmarker.linux_benchmarks import nginx_benchmark
-from perfkitbenchmarker.resources.container_service import kubernetes_commands
 
 FLAGS = flags.FLAGS
 
+flags.DEFINE_string(
+    'nginx_conf',
+    None,
+    'Path to a custom nginx configuration file.',
+)
+
 flags.DEFINE_string(
     'kubernetes_nginx_runtime_class_name',
     None,
@@ -48,7 +56,16 @@
     vm_spec: *default_dual_core
     nodepools:
       nginx:
-        vm_count: 3
+        vm_count: 1
+        vm_spec:
+          GCP:
+            machine_type: n2-standard-4
+          AWS:
+            machine_type: m6i.xlarge
+          Azure:
+            machine_type: Standard_D4s_v5
+      upstream:
+        vm_count: 2
         vm_spec:
           GCP:
             machine_type: n2-standard-4
@@ -95,49 +112,159 @@ def GetConfig(user_config):
   return config
 
 
+def _MergeNginxConfigs(global_conf_path, server_conf_path, force_http=False):
+  """Merges global and server nginx configs into a single file."""
+  with open(global_conf_path) as f:
+    global_conf = f.read()
+  with open(server_conf_path) as f:
+    server_conf = f.read()
+
+  # Replace placeholder with K8s upstream service DNS
+  # Upstream service name is 'nginx-upstream' in 'default' namespace
+  upstream_dns = 'nginx-upstream.default.svc.cluster.local'
+  # Hardcode Upstream Port to 80 (Profile 2 / Single TLS Parity)
+  upstream_port = 80
+  
+  # The placeholder in rp_apigw.conf includes :443;
+  # We replace the first occurrence (fileserver_1)
+  server_conf = server_conf.replace(
+      '# server <fileserver_1_ip_or_dns>:443;',
+      f'server {upstream_dns}:{upstream_port};',
+  )
+
+  if (not FLAGS.nginx_use_ssl) or force_http:
+    # Convert HTTPS config to HTTP
+    server_conf = server_conf.replace('ssl on;', '# ssl on;')
+    server_conf = server_conf.replace('ssl_certificate', '# ssl_certificate')
+    server_conf = server_conf.replace('ssl_ciphers', '# ssl_ciphers')
+    server_conf = server_conf.replace('listen 443 ssl', 'listen 80')
+  
+  # ALWAYS force upstream connection to HTTP (Profile 2)
+  # This ensures Proxy -> Upstream is unencrypted even if Proxy listener is HTTPS
+  server_conf = server_conf.replace('proxy_pass https://', 'proxy_pass http://')
+
+  # Simple merge: replace the include line with the actual content
+  # global.conf has: include /etc/nginx/conf.d/*.conf;
+  merged_conf = global_conf.replace(
+      'include /etc/nginx/conf.d/*.conf;', server_conf
+  )
+  return merged_conf
+
+
 def _CreateNginxConfigMapDir():
   """Returns a TemporaryDirectory containing files in the Nginx ConfigMap."""
+  temp_dir = tempfile.TemporaryDirectory()
+
+  # 1. Get paths to source config files
+  global_conf_path = data.ResourcePath('nginx/global.conf')
+  proxy_conf_path = data.ResourcePath('nginx/rp_apigw.conf')
+  upstream_conf_path = data.ResourcePath('nginx/file_server.conf')
+
   if FLAGS.nginx_conf:
-    nginx_conf_filename = FLAGS.nginx_conf
-  else:
-    relative_nginx_conf_filename = 'container/kubernetes_nginx/http.conf'
-    if FLAGS.nginx_use_ssl:
-      relative_nginx_conf_filename = 'container/kubernetes_nginx/https.conf'
-    nginx_conf_filename = data.ResourcePath(relative_nginx_conf_filename)
+    # If custom config provided, use it for proxy (legacy behavior support)
+    # But for 3-tier, we really need the split configs.
+    # For now, let's assume nginx_conf overrides the proxy config if given.
+    proxy_conf_path = FLAGS.nginx_conf
+
+  # 2. Prepare Proxy Config (global + rp_apigw)
+  proxy_conf_content = _MergeNginxConfigs(global_conf_path, proxy_conf_path)
+  with open(os.path.join(temp_dir.name, 'nginx-proxy.conf'), 'w') as f:
+    f.write(proxy_conf_content)
+
+  # 3. Prepare Upstream Config (global + file_server)
+  # Force HTTP for Upstream (Profile 2) similar to the GCE fix
+  upstream_conf_content = _MergeNginxConfigs(
+      global_conf_path, upstream_conf_path, force_http=True
+  )
+  with open(os.path.join(temp_dir.name, 'nginx-upstream.conf'), 'w') as f:
+    f.write(upstream_conf_content)
 
-  temp_dir = tempfile.TemporaryDirectory()
-  config_map_filename = os.path.join(temp_dir.name, 'default')
-  shutil.copyfile(nginx_conf_filename, config_map_filename)
   return temp_dir
 
 
+def _WaitForConnectivity(benchmark_spec):
+  """Waits for the proxy to be reachable."""
+  lb_ip = benchmark_spec.nginx_endpoint_ip
+  logging.info('Waiting for connectivity to %s...', lb_ip)
+
+  # Try to connect to the endpoint
+  scheme = 'https' if FLAGS.nginx_use_ssl else 'http'
+  url = f'{scheme}://{lb_ip}/'
+
+  # Use curl to check connectivity
+  cmd = f'curl -k -v {url}'
+  
+  # Retry for up to 5 minutes
+  start_time = time.time()
+  while time.time() - start_time < 300:
+    try:
+      # We run this from the client VM to verify end-to-end connectivity
+      benchmark_spec.vm_groups['clients'][0].RemoteCommand(cmd)
+      logging.info('Connectivity established.')
+      return
+    except errors.VirtualMachine.RemoteCommandError:
+      logging.info('Still waiting for connectivity...')
+      time.sleep(10)
+  
+  raise errors.Benchmarks.PrepareException(
+      f'Timed out waiting for connectivity to {url}'
+  )
+
+
 def _PrepareCluster(benchmark_spec):
   """Prepares a cluster to run the Nginx benchmark."""
+  # 1. Create ConfigMap
   with _CreateNginxConfigMapDir() as nginx_config_map_dirname:
     benchmark_spec.container_cluster.CreateConfigMap(
-        'default-config', nginx_config_map_dirname
+        'nginx-configs', nginx_config_map_dirname
     )
-  container_image = benchmark_spec.container_specs['kubernetes_nginx'].image
-  replicas = benchmark_spec.container_cluster.nodepools['nginx'].num_nodes
 
-  nginx_port = 80
-  if FLAGS.nginx_use_ssl:
-    nginx_port = 443
+  container_image = benchmark_spec.container_specs['kubernetes_nginx'].image
+  proxy_port = 443 if FLAGS.nginx_use_ssl else 80
+  upstream_port = 80 # Hardcoded for Profile 2
 
-  kubernetes_commands.ApplyManifest(
-      'container/kubernetes_nginx/kubernetes_nginx.yaml.j2',
+  # 2. Deploy Upstream
+  # We use the 'upstream' nodepool
+  upstream_replicas = benchmark_spec.container_cluster.nodepools[
+      'upstream'
+  ].num_nodes
+  benchmark_spec.container_cluster.ApplyManifest(
+      'container/kubernetes_nginx/nginx_upstream.yaml.j2',
       nginx_image=container_image,
-      nginx_replicas=replicas,
+      nginx_upstream_replicas=upstream_replicas,
       nginx_content_size=FLAGS.nginx_content_size,
-      nginx_port=nginx_port,
-      nginx_worker_connections=FLAGS.nginx_worker_connections,
+      nginx_port=upstream_port,
       runtime_class_name=FLAGS.kubernetes_nginx_runtime_class_name,
   )
 
+  # 3. Deploy Proxy
+  # We use the 'nginx' nodepool (renamed to proxy in our minds, but key is 'nginx')
+  proxy_replicas = benchmark_spec.container_cluster.nodepools['nginx'].num_nodes
+  benchmark_spec.container_cluster.ApplyManifest(
+      'container/kubernetes_nginx/nginx_proxy.yaml.j2',
+      nginx_image=container_image,
+      nginx_proxy_replicas=proxy_replicas,
+      nginx_port=proxy_port,
+      runtime_class_name=FLAGS.kubernetes_nginx_runtime_class_name,
+  )
+
+  # 4. Wait for deployments
+  benchmark_spec.container_cluster.WaitForResource(
+      'deploy/nginx-upstream-deployment', 'available'
+  )
   benchmark_spec.container_cluster.WaitForResource(
-      'deploy/nginx-deployment', 'available'
+      'deploy/nginx-proxy-deployment', 'available'
   )
 
+  # 5. Get LoadBalancer IP
+  # Get LoadBalancer IP using PKB's built-in retry method
+  benchmark_spec.nginx_endpoint_ip = (
+      benchmark_spec.container_cluster.GetLoadBalancerIP('nginx-cluster')
+  )
+
+  # 6. Wait for connectivity
+  _WaitForConnectivity(benchmark_spec)
+
 
 def Prepare(benchmark_spec):
   """Install Nginx on the K8s Cluster and a load generator on the clients.
@@ -154,10 +281,7 @@ def Prepare(benchmark_spec):
 
   background_tasks.RunThreaded(lambda f: f(), prepare_fns)
 
-  benchmark_spec.nginx_endpoint_ip = (
-      benchmark_spec.container_cluster.GetClusterIP('nginx-cluster')
-  )
-
+  # benchmark_spec.nginx_endpoint_ip is set in _PrepareCluster
 
 def Run(benchmark_spec):
   """Run a benchmark against the Nginx server."""
