ghx-multi.yaml

---
# PersistentVolumeClaim for 数据存储
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ghx-pvc
  namespace: gpu-health-expert
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: <your-storage-class-name>
---
# ConfigMap for frontend environment variables
apiVersion: v1
kind: ConfigMap
metadata:
  name: pb-env-config
  namespace: gpu-health-expert
# 后端service
data:
  env.js: |
    window.NEXT_PUBLIC_API_URL='<your-backend-service-url>';
---
# Job模板ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: job-template-config
  namespace: gpu-health-expert
data:
  # 烧机测试Job模板
  burnin-job-template.yaml: |
    apiVersion: batch/v1
    kind: Job
    metadata:
      name: ghx-burnin-job-{JOB_ID}
      namespace: gpu-health-expert 
      labels:
        app: ghx-burnin
        job-type: burnin
        job-id: "{BASE_JOB_ID}"
        created-by: ghx-server
    spec:
      parallelism: 1
      completions: 1
      ttlSecondsAfterFinished: 300  # Job完成后5分钟自动销毁
      backoffLimit: 0  # 禁止Job重试
      template:
        metadata:
          labels:
            app: ghx-burnin
            job-type: burnin
            job-id: "{BASE_JOB_ID}"
        spec:
          restartPolicy: Never
          tolerations:
          - operator: "Exists"
            effect: "NoSchedule"
          - operator: "Exists"
            effect: "PreferNoSchedule"
          - operator: "Exists"
            effect: "NoExecute"
          nodeSelector:
            kubernetes.io/hostname: "{NODE_NAME}"  # 直接指定前端选择的节点
        containers:
        - name: gpu-burnin
          image: kang2023/ghx-check:latest
          imagePullPolicy: IfNotPresent
          workingDir: /opt/gpu-burn
          command: ["python3"]
          args: ["/opt/gpu_burnin_monitor.py"]
            env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: JOB_ID
              value: "{JOB_ID}"
            - name: MEMORY_PARAM
              value: "{MEMORY_PARAM}"
            - name: DURATION
              value: "{DURATION}"
          - name: API_SERVER_URL
            value: "http://ghx-service.gpu-health-expert.svc.cluster.local:5000"
            resources:
              requests:
                {GPU_RESOURCE_NAME}: 8  # 8个GPU
              limits:
                {GPU_RESOURCE_NAME}: 8  # 8个GPU
            volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
          volumes:
          - name: localtime
            hostPath:
              path: /etc/localtime
              type: File
  # 诊断测试Job模板
  job-template.yaml: |
    apiVersion: batch/v1
    kind: Job
    metadata:
      name: ghx-manual-job-{JOB_ID}
      namespace: gpu-health-expert 
      labels:
        app: ghx-manual
        job-type: manual
        job-id: "{BASE_JOB_ID}"
        created-by: gpu-cli
    spec:
      parallelism: 1
      completions: 1
      ttlSecondsAfterFinished: 60  # Job完成后1分钟自动销毁
      backoffLimit: 0  # 禁止Job重试
      template:
        metadata:
          labels:
            app: ghx-manual
            job-type: manual
            job-id: "{BASE_JOB_ID}"
        spec:
          restartPolicy: Never
          tolerations:
          - operator: "Exists"
            effect: "NoSchedule"
          - operator: "Exists"
            effect: "PreferNoSchedule"
          - operator: "Exists"
            effect: "NoExecute"
          nodeSelector:
            kubernetes.io/hostname: "{NODE_NAME}"  # 直接指定前端选择的节点
          containers:
          - name: gpu-checker
            image: kang2023/ghx-check:latest
            imagePullPolicy: IfNotPresent
            command: ["python3"]
            args: ["/opt/gpu_check_job.py"]
            env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: JOB_TYPE
              value: "manual"
            - name: JOB_ID
              value: "{JOB_ID}"
            - name: ENABLED_TESTS
              value: "{ENABLED_TESTS}"
            - name: DCGM_DIAG_LEVEL
              value: "{DCGM_LEVEL}"
            - name: SELECTED_NODES
              value: "{SELECTED_NODES}"
            resources:
              requests:
                {GPU_RESOURCE_NAME}: 8
                {RDMA_RESOURCES}
              limits:
                {GPU_RESOURCE_NAME}: 8
                {RDMA_RESOURCES}
            securityContext:
              privileged: true
            volumeMounts:
            - name: shared-results
              mountPath: /shared
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: gpu-benchmarks-config
              mountPath: /config/gpu-benchmarks.json
              subPath: gpu-benchmarks.json
          volumes:
          - name: shared-results
            persistentVolumeClaim:
              claimName: ghx-pvc
          - name: localtime
            hostPath:
              path: /etc/localtime
              type: File
          - name: gpu-benchmarks-config
            configMap:
              name: gpu-benchmarks-config
---
# GPU基准值ConfigMap（前后端通用）
apiVersion: v1
kind: ConfigMap
metadata:
  name: gpu-benchmarks-config
  namespace: gpu-health-expert
data:
  gpu-benchmarks.json: |
    {
      "RTX 3090": {"p2p": 18, "nccl": 7, "bw": 20},
      "L40S": {"p2p": 28, "nccl": 8, "bw": 20},
      "RTX 4090": {"p2p": 18, "nccl": 7, "bw": 20},
      "A100": {"p2p": 420, "nccl": 70, "bw": 20},
      "A800": {"p2p": 340, "nccl": 55, "bw": 20},
      "H100": {"p2p": 700, "nccl": 139, "bw": 40},
      "H800": {"p2p": 340, "nccl": 65, "bw": 47},
      "H20": {"p2p": 700, "nccl": 139, "bw": 47},
      "H200": {"p2p": 730, "nccl": 145, "bw": 54}
    }
  env.js: |
    window.GPU_BENCHMARKS = {
      "RTX 3090": {"p2p": 18, "nccl": 7, "bw": 20},
      "L40S": {"p2p": 28, "nccl": 8, "bw": 20},
      "RTX 4090": {"p2p": 18, "nccl": 7, "bw": 20},
      "A100": {"p2p": 420, "nccl": 70, "bw": 20},
      "A800": {"p2p": 340, "nccl": 55, "bw": 20},
      "H100": {"p2p": 700, "nccl": 139, "bw": 40},
      "H800": {"p2p": 340, "nccl": 65, "bw": 47},
      "H20": {"p2p": 700, "nccl": 139, "bw": 47},
      "H200": {"p2p": 730, "nccl": 145, "bw": 54}
    }
---
# GPU检查CronJob - 在每个GPU节点上运行
## L40s
apiVersion: batch/v1
kind: CronJob
metadata:
  name: ghx-cronjob-l40s
  namespace: gpu-health-expert
spec:
  schedule: "0 4 * * *"  # 每天凌晨4点执行
  concurrencyPolicy: Allow  # 允许并发执行
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        job-type: cron
    spec:
      parallelism: 5    # 最大并发 Job 数
      completions: 5    # 总需完成的 Job 数（假设有 10 个节点）
      ttlSecondsAfterFinished: 60  # Job完成后1分钟自动销毁
      backoffLimit: 0 # 禁止 Job 重试
      template:
        spec:
          restartPolicy: Never
          tolerations:
          - operator: "Exists"
            effect: "NoSchedule"
          - operator: "Exists"
            effect: "PreferNoSchedule"
          - operator: "Exists"
            effect: "NoExecute"
          containers:
          - name: gpu-checker
            image: kang2023/ghx-check:latest  # 使用我们的GPU检查镜像
            imagePullPolicy: IfNotPresent
            command: ["python3"]
            args: ["/opt/gpu_check_job.py"]
            env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: DCGM_DIAG_LEVEL
              value: "1"  # DCGM诊断级别，可通过环境变量覆盖
            - name: JOB_TYPE
              value: "cron"
            resources:
              requests:
                nvidia.com/gpu-l40s: 8
              limits:
                nvidia.com/gpu-l40s: 8 
            securityContext:
              privileged: true
            volumeMounts:
            - name: shared-results
              mountPath: /shared
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: gpu-benchmarks-config
              mountPath: /config/gpu-benchmarks.json
              subPath: gpu-benchmarks.json
          volumes:
          - name: shared-results
            persistentVolumeClaim:
              claimName: ghx-pvc
          - name: localtime
            hostPath:
              path: /etc/localtime
              type: File
          - name: gpu-benchmarks-config
            configMap:
              name: gpu-benchmarks-config
---
## h100-80gb-hbm3
apiVersion: batch/v1
kind: CronJob
metadata:
  name: ghx-cronjob-h100-80gb-hbm3
  namespace: gpu-health-expert
spec:
  schedule: "0 4 * * *"  # 每天凌晨4点执行
  concurrencyPolicy: Allow  # 允许并发执行
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        job-type: cron
    spec:
      parallelism: 5    # 最大并发 Job 数
      completions: 5    # 总需完成的 Job 数（假设有 10 个节点）
      ttlSecondsAfterFinished: 60  # Job完成后1分钟自动销毁
      backoffLimit: 0 # 禁止 Job 重试
      template:
        spec:
          restartPolicy: Never
          tolerations:
          - operator: "Exists"
            effect: "NoSchedule"
          - operator: "Exists"
            effect: "PreferNoSchedule"
          - operator: "Exists"
            effect: "NoExecute"
          containers:
          - name: gpu-checker
            image: kang2023/ghx-check:latest  # 使用我们的GPU检查镜像
            imagePullPolicy: IfNotPresent
            command: ["python3"]
            args: ["/opt/gpu_check_job.py"]
            env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: DCGM_DIAG_LEVEL
              value: "1"  # DCGM诊断级别，可通过环境变量覆盖
            - name: JOB_TYPE
              value: "cron"
            resources:
              requests:
                nvidia.com/gpu-h100-80gb-hbm3: 8
                rdma/rdma_shared_device_a: 1
                rdma/rdma_shared_device_b: 1
              limits:
                nvidia.com/gpu-h100-80gb-hbm3: 8
                rdma/rdma_shared_device_a: 1
                rdma/rdma_shared_device_b: 1
            securityContext:
              privileged: true
            volumeMounts:
            - name: shared-results
              mountPath: /shared
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: gpu-benchmarks-config
              mountPath: /config/gpu-benchmarks.json
              subPath: gpu-benchmarks.json
          volumes:
          - name: shared-results
            persistentVolumeClaim:
              claimName: ghx-pvc
          - name: localtime
            hostPath:
              path: /etc/localtime
              type: File
          - name: gpu-benchmarks-config
            configMap:
              name: gpu-benchmarks-config
---
## h100
apiVersion: batch/v1
kind: CronJob
metadata:
  name: ghx-cronjob-h100
  namespace: gpu-health-expert
spec:
  schedule: "0 4 * * *"  # 每天凌晨4点执行
  concurrencyPolicy: Allow  # 允许并发执行
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        job-type: cron
    spec:
      parallelism: 5    # 最大并发 Job 数
      completions: 5    # 总需完成的 Job 数（假设有 10 个节点）
      ttlSecondsAfterFinished: 60  # Job完成后1分钟自动销毁
      backoffLimit: 0 # 禁止 Job 重试
      template:
        spec:
          restartPolicy: Never
          tolerations:
          - operator: "Exists"
            effect: "NoSchedule"
          - operator: "Exists"
            effect: "PreferNoSchedule"
          - operator: "Exists"
            effect: "NoExecute"
          containers:
          - name: gpu-checker
            image: kang2023/ghx-check:latest  # 使用我们的GPU检查镜像
            imagePullPolicy: IfNotPresent
            command: ["python3"]
            args: ["/opt/gpu_check_job.py"]
            env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: DCGM_DIAG_LEVEL
              value: "1"  # DCGM诊断级别，可通过环境变量覆盖
            - name: JOB_TYPE
              value: "cron"
            resources:
              requests:
                nvidia.com/gpu-h100: 8
                rdma/rdma_shared_device_a: 1
                rdma/rdma_shared_device_b: 1
              limits:
                nvidia.com/gpu-h100: 8
                rdma/rdma_shared_device_a: 1
                rdma/rdma_shared_device_b: 1
            securityContext:
              privileged: true
            volumeMounts:
            - name: shared-results
              mountPath: /shared
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: gpu-benchmarks-config
              mountPath: /config/gpu-benchmarks.json
              subPath: gpu-benchmarks.json
          volumes:
          - name: shared-results
            persistentVolumeClaim:
              claimName: ghx-pvc
          - name: localtime
            hostPath:
              path: /etc/localtime
              type: File
          - name: gpu-benchmarks-config
            configMap:
              name: gpu-benchmarks-config
---
## h800
apiVersion: batch/v1
kind: CronJob
metadata:
  name: ghx-cronjob-h800
  namespace: gpu-health-expert
spec:
  schedule: "0 4 * * *"  # 每天凌晨4点执行
  concurrencyPolicy: Allow  # 允许并发执行
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    metadata:
      labels:
        job-type: cron
    spec:
      parallelism: 5    # 最大并发 Job 数
      completions: 5    # 总需完成的 Job 数（假设有 10 个节点）
      ttlSecondsAfterFinished: 60  # Job完成后1分钟自动销毁
      backoffLimit: 0 # 禁止 Job 重试
      template:
        spec:
          restartPolicy: Never
          tolerations:
          - operator: "Exists"
            effect: "NoSchedule"
          - operator: "Exists"
            effect: "PreferNoSchedule"
          - operator: "Exists"
            effect: "NoExecute"
          containers:
          - name: gpu-checker
            image: kang2023/ghx-check:latest  # 使用我们的GPU检查镜像
            imagePullPolicy: IfNotPresent
            command: ["python3"]
            args: ["/opt/gpu_check_job.py"]
            env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: DCGM_DIAG_LEVEL
              value: "1"  # DCGM诊断级别，可通过环境变量覆盖
            - name: JOB_TYPE
              value: "cron"
            resources:
              requests:
                nvidia.com/gpu-h800: 8
                rdma/rdma_shared_device_a: 1
                rdma/rdma_shared_device_b: 1
              limits:
                nvidia.com/gpu-h800: 8
                rdma/rdma_shared_device_a: 1
                rdma/rdma_shared_device_b: 1
            securityContext:
              privileged: true
            volumeMounts:
            - name: shared-results
              mountPath: /shared
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: gpu-benchmarks-config
              mountPath: /config/gpu-benchmarks.json
              subPath: gpu-benchmarks.json
          volumes:
          - name: shared-results
            persistentVolumeClaim:
              claimName: ghx-pvc
          - name: localtime
            hostPath:
              path: /etc/localtime
              type: File
          - name: gpu-benchmarks-config
            configMap:
              name: gpu-benchmarks-config
---
# GHX统一服务Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ghx-server
  namespace: gpu-health-expert
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ghx-server
  template:
    metadata:
      labels:
        app: ghx-server
    spec:
      tolerations:
      - operator: "Exists"
        effect: "NoSchedule"
      - operator: "Exists"
        effect: "PreferNoSchedule"
      - operator: "Exists"
        effect: "NoExecute"
      containers:
      - name: ghx-server
        image: kang2023/ghx-server:latest  # GHX统一服务镜像
        imagePullPolicy: IfNotPresent
        env:
        - name: GPU_RESULT_RETENTION_DAYS
          value: "7"  # GPU结果保留天数，默认为7天
        - name: PORT
          value: "5000"
        - name: KUBECONFIG
          value: "/root/.kube/config"
        - name: CORS_ORIGINS
          value: "http://<your-server-ip>:<service-port>"
        ports:
        - containerPort: 5000
        volumeMounts:
        - name: shared-results
          mountPath: /shared
        - name: kubeconfig
          mountPath: /root/.kube
          readOnly: true
        - name: localtime
          mountPath: /etc/localtime
          readOnly: true
        - name: job-template-config
          mountPath: /app/job-template.yaml
          subPath: job-template.yaml
          readOnly: true
        - name: job-template-config
          mountPath: /app/burnin-job-template.yaml
          subPath: burnin-job-template.yaml
          readOnly: true
      volumes:
      - name: shared-results
        persistentVolumeClaim:
          claimName: ghx-pvc
      - name: kubeconfig
        secret:
          secretName: gpu-health-expert-kubeconfig
          defaultMode: 0400
      - name: localtime
        hostPath:
          path: /etc/localtime
          type: File
      - name: job-template-config
        configMap:
          name: job-template-config
---
# GHX统一服务Service
apiVersion: v1
kind: Service
metadata:
  name: ghx-service
  namespace: gpu-health-expert
spec:
  selector:
    app: ghx-server
  ports:
  - name: http
    port: 5000
    targetPort: 5000
    nodePort: 31005
  type: NodePort 
---
# GHX前端Dashboard Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ghx-dashboard
  namespace: gpu-health-expert
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ghx-dashboard
  template:
    metadata:
      labels:
        app: ghx-dashboard
    spec:
      containers:
      - name: ghx-dashboard
        image: kang2023/ghx-dashboard:latest  # GHX前端Dashboard镜像
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 3000
        volumeMounts:
        - name: pb-env
          mountPath: /app/public/env.js
          subPath: env.js
        - name: gpu-benchmarks-env
          mountPath: /app/public/gpu-benchmarks.js
          subPath: env.js
        - name: localtime
          mountPath: /etc/localtime
          readOnly: true
      volumes:
      - name: pb-env
        configMap:
          name: pb-env-config
      - name: gpu-benchmarks-env
        configMap:
          name: gpu-benchmarks-config
      - name: localtime
        hostPath:
          path: /etc/localtime
          type: File
      tolerations:
      - operator: "Exists"
        effect: "NoSchedule"
      - operator: "Exists"
        effect: "PreferNoSchedule"
      - operator: "Exists"
        effect: "NoExecute"
---
# GHX前端Dashboard Service
apiVersion: v1
kind: Service
metadata:
  name: ghx-dashboard-service
  namespace: gpu-health-expert
spec:
  selector:
    app: gpu-dashboard
  ports:
  - name: http
    port: 80
    targetPort: 3000
    nodePort: 31033
  type: NodePort