From efb28a75e07edd1779cfe7a7b50b1e85cbbd8574 Mon Sep 17 00:00:00 2001 From: huizhang Date: Mon, 23 Mar 2026 18:15:32 +0800 Subject: [PATCH 1/2] add how to cleanup cluster image registry --- ...ster_Image_Registry_Administrator_Guide.md | 410 ++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100644 docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md diff --git a/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md b/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md new file mode 100644 index 00000000..80b6560e --- /dev/null +++ b/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md @@ -0,0 +1,410 @@ +--- +products: + - Alauda Container Platform +kind: + - Solution +ProductsVersion: + - 4.x +--- + +# Administrator Guide to Cluster Image Registry Cleanup and Scheduled Task Configuration + +## Introduction + +This document provides a practical and production-ready approach for cleaning images in ACP's internal registry of cluster and automating the cleanup process with `CronJob`. + +Goals of this solution: + +- Safely remove images that are no longer used by the cluster. +- Control cleanup behavior with retention time, revision count, and whitelist rules. +- Start with Dry Run, then execute confirmed cleanup. +- Trigger registry garbage collection (GC) when needed. +- Configure scheduled tasks for automatic recurring cleanup. + +## Prerequisites + +- ACP CLI (`ac`) is installed. +- You can access the target cluster. +- You can access the registry. +- If running in Pod/CronJob: + - `serviceAccountName` must be configured. + - The ServiceAccount must have permissions to list workloads and access the registry path. + +## Architecture Overview + +```text +┌────────────────────┐ +│ CronJob / Job / Pod│ +│ ac adm prune images│ +└─────────┬──────────┘ + │ + │ scan in-use images + ▼ +┌────────────────────┐ +│ Kubernetes API │ +│ Pods/Deployments...│ +└─────────┬──────────┘ + │ + │ fetch registry metadata + ▼ +┌────────────────────┐ +│ Image Registry API │ +│ repos/tags/digests │ +└─────────┬──────────┘ + │ + │ filter and delete manifests + ▼ +┌────────────────────┐ +│ Prune Result │ +│ (Optional GC) │ +└────────────────────┘ +``` + +## Parameter Reference + +All parameters are optional. If you run the command without any parameters, it only outputs candidate images and does not delete anything. + +| Parameter | Purpose | Typical example | +|-----------|---------|-----------------| +| `--keep-younger-than=` | Keep recently created images | `168h` | +| `--keep-tag-revisions=` | Keep latest N versions per repository | `5` | +| `--all` | Ignore retention time and revision count, prune all unused images | `--all` | +| `--whitelist=` | Exclude matching repositories from pruning (repeatable) | `^cpaas-system/.*` | +| `--dry-run` | Show candidates only, no deletion (default) | `--dry-run` | +| `--confirm` | Execute actual deletion | `--confirm` | +| `--prune-registry` | Trigger registry GC after deletion | `--prune-registry` | +| `--registry-url=` | Set registry endpoint | `http://image-registry.cpaas-system` | + +## Implementation Steps + +### Step 1: Log in and select target cluster + +```bash +ac login +ac config use-cluster +``` + +### Step 2: Run Dry Run first + +```bash +ac adm prune images +``` + +### Step 3: Run confirmed cleanup with policy + +```bash +ac adm prune images \ + --keep-younger-than=168h \ + --keep-tag-revisions=5 \ + --whitelist='^cpaas-system/.*' \ + --confirm +``` + +### Step 4: Trigger GC when required + +```bash +ac adm prune images \ + --keep-younger-than=72h \ + --keep-tag-revisions=3 \ + --prune-registry \ + --confirm +``` + +## Configure Scheduled Cleanup with CronJob + +### Base CronJob Template + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ac-prune-images-cronjob + namespace: cpaas-system +spec: + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + serviceAccountName: ac-images-pruner-sa + restartPolicy: Never + containers: + - name: ac + image: /alauda/ac: + args: + - adm + - prune + - images +``` + +Notes: +- ``: registry endpoint of your target ACP environment. +- ``: AC image tag from your target ACP environment. + +### Fully Runnable Example (Dry Run Recommended) + +The following resources include the minimal runnable set and can be copied directly. + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ac-images-pruner-sa + namespace: cpaas-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ac-images-pruner-read +rules: + - apiGroups: [""] + resources: ["pods", "replicationcontrollers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets", "daemonsets", "replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ac-images-pruner-read-binding +subjects: + - kind: ServiceAccount + name: ac-images-pruner-sa + namespace: cpaas-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ac-images-pruner-read +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ac-prune-images-cronjob + namespace: cpaas-system +spec: + schedule: "0 */6 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + serviceAccountName: ac-images-pruner-sa + restartPolicy: Never + containers: + - name: ac + image: /alauda/ac: + args: + - adm + - prune + - images + - --keep-younger-than=168h + - --keep-tag-revisions=5 + - --whitelist=^cpaas-system/.* +``` + +Apply resources: + +```bash +ac apply -f ac-prune-images-cronjob.yaml +``` + +Trigger one manual Job for immediate validation: + +```bash +ac create job --from=cronjob/ac-prune-images-cronjob \ + ac-prune-images-cronjob-manual -n cpaas-system +``` + +Check execution result: + +```bash +ac get job -n cpaas-system ac-prune-images-cronjob-manual +``` + +## Scenario-Based Demos + +### Scenario 1: Daily inspection (Dry Run only) + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ac-prune-images-daily + namespace: cpaas-system +spec: + schedule: "0 2 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + serviceAccountName: ac-tester-sa + restartPolicy: Never + containers: + - name: ac + image: /alauda/ac: + args: + - adm + - prune + - images + - --keep-younger-than=168h + - --keep-tag-revisions=5 + - --whitelist=^cpaas-system/.* +``` + +### Scenario 2: Weekly steady cleanup (recommended for production) + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ac-prune-images-weekly + namespace: cpaas-system +spec: + schedule: "30 3 * * 0" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + serviceAccountName: ac-tester-sa + restartPolicy: Never + containers: + - name: ac + image: /alauda/ac: + args: + - adm + - prune + - images + - --keep-younger-than=336h + - --keep-tag-revisions=10 + - --whitelist=^cpaas-system/.* + - --confirm +``` + +### Scenario 3: Monthly aggressive cleanup (with whitelist protection) + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ac-prune-images-monthly-aggressive + namespace: cpaas-system +spec: + schedule: "0 4 1 * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + template: + spec: + serviceAccountName: ac-tester-sa + restartPolicy: Never + containers: + - name: ac + image: /alauda/ac: + args: + - adm + - prune + - images + - --all + - --whitelist=^cpaas-system/.* + - --whitelist=^pro-ns1/base/.* + - --confirm +``` + +### Scenario 4: Weekly cleanup with GC (off-peak window) + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ac-prune-images-with-gc + namespace: cpaas-system +spec: + schedule: "0 1 * * 6" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + template: + spec: + serviceAccountName: ac-tester-sa + restartPolicy: Never + containers: + - name: ac + image: /alauda/ac: + args: + - adm + - prune + - images + - --keep-younger-than=720h + - --keep-tag-revisions=5 + - --prune-registry + - --confirm +``` + +## Verification + +Run these commands after deployment: + +```bash +ac get cronjob -n cpaas-system +ac get job -n cpaas-system +ac get pod -n cpaas-system +``` + +Expected result: + +- Pod status is `Completed`. +- `exitCode` is `0`. +- No restart loop under `restartPolicy: Never`. + +Check logs for details: + +```bash +ac logs job/ac-prune-images-daily -n cpaas-system +``` + +Typical successful `Pod` logs (example): + +```text +[1/5] Scanning cluster for used images... + Found 75 unique image references in use. +[2/5] Fetching metadata from registry... + Scanned 9 repositories, found 1 image instances. +[3/5] Pruning 1 image manifests... + DRY RUN: Would delete pro-ns1/demo/bash:5 +[4/5] Summary + Candidates for pruning: 1 +[5/5] Dry run mode, skipping trigger registry garbage collection. +``` + +## Troubleshooting + +| Symptom | Possible cause | Suggested action | +|---------|----------------|------------------| +| `no current context set` | Image does not include in-cluster fallback and no `ac login` session exists in container | Upgrade to a compatible AC version and verify SA token mount | +| `forbidden` / `cannot list ...` | ServiceAccount lacks RBAC for scan resources | Add required list/get/watch permissions to `ClusterRole` | +| `failed to list registry pods` | No access to registry Pod in `cpaas-system` | Verify RBAC and namespace settings | +| Registry auth-related errors | Registry auth policy does not match current token mode | Verify `--registry-url` and registry token/anonymous policy | +| No prune candidates | Retention settings are too strict or whitelist is too broad | Run Dry Run and tune `--keep-*` and `--whitelist` | From a929736778f055518a7e1bedefe31c41c38de71a Mon Sep 17 00:00:00 2001 From: huizhang Date: Tue, 24 Mar 2026 18:35:30 +0800 Subject: [PATCH 2/2] add how to cleanup cluster image registry --- ...ster_Image_Registry_Administrator_Guide.md | 151 +++++++++++++++--- 1 file changed, 127 insertions(+), 24 deletions(-) diff --git a/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md b/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md index 80b6560e..d6f98c74 100644 --- a/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md +++ b/docs/en/solutions/How_to_Cleanup_Alauda_Container_Platform_Cluster_Image_Registry_Administrator_Guide.md @@ -11,7 +11,7 @@ ProductsVersion: ## Introduction -This document provides a practical and production-ready approach for cleaning images in ACP's internal registry of cluster and automating the cleanup process with `CronJob`. +This document provides a practical and production-ready approach for cleaning images in ACP's internal cluster image registry and automating the cleanup process with `CronJob`. Goals of this solution: @@ -28,7 +28,7 @@ Goals of this solution: - You can access the registry. - If running in Pod/CronJob: - `serviceAccountName` must be configured. - - The ServiceAccount must have permissions to list workloads and access the registry path. + - The ServiceAccount must have permissions to list workloads, access registry APIs, and (if GC is enabled) access the `image-registry` Pod exec endpoint. ## Architecture Overview @@ -72,7 +72,7 @@ All parameters are optional. If you run the command without any parameters, it o | `--whitelist=` | Exclude matching repositories from pruning (repeatable) | `^cpaas-system/.*` | | `--dry-run` | Show candidates only, no deletion (default) | `--dry-run` | | `--confirm` | Execute actual deletion | `--confirm` | -| `--prune-registry` | Trigger registry GC after deletion | `--prune-registry` | +| `--prune-registry` | Trigger registry GC in non-dry-run mode, even if no manifests were deleted | `--prune-registry` | | `--registry-url=` | Set registry endpoint | `http://image-registry.cpaas-system` | ## Implementation Steps @@ -81,6 +81,7 @@ All parameters are optional. If you run the command without any parameters, it o ```bash ac login +ac config get-clusters ac config use-cluster ``` @@ -90,8 +91,16 @@ ac config use-cluster ac adm prune images ``` +If `ac` is not able to detect the internal registry endpoint automatically, you can specify the external registry endpoint with the `--registry-url` parameter. + +```bash +ac adm prune images --registry-url= +``` + ### Step 3: Run confirmed cleanup with policy +Keep younger than 7 days, keep latest 5 revisions per repository, and exclude images in `cpaas-system` namespace, confirm the cleanup operation. + ```bash ac adm prune images \ --keep-younger-than=168h \ @@ -102,6 +111,8 @@ ac adm prune images \ ### Step 4: Trigger GC when required +Keep younger than 3 days, keep latest 3 revisions per repository, and trigger registry GC in non-dry-run mode, confirm the cleanup operation. + ```bash ac adm prune images \ --keep-younger-than=72h \ @@ -114,6 +125,8 @@ ac adm prune images \ ### Base CronJob Template +Run image-prune inspection against `image-registry.cpaas-system` once per day at 2:00 AM using a CronJob (dry-run by default). + ```yaml apiVersion: batch/v1 kind: CronJob @@ -133,7 +146,7 @@ spec: restartPolicy: Never containers: - name: ac - image: /alauda/ac: + image: /alauda/ac: args: - adm - prune @@ -141,12 +154,13 @@ spec: ``` Notes: -- ``: registry endpoint of your target ACP environment. -- ``: AC image tag from your target ACP environment. +- ``: registry endpoint of your target ACP platform. +- ``: AC image tag from your target ACP platform. +- `serviceAccountName: ac-images-pruner-sa`: ServiceAccount must have permissions to list workloads, access the `image-registry` Pod (for GC), and access `registry.alauda.io/images` (`get` for dry-run, `delete` for confirmed cleanup). ### Fully Runnable Example (Dry Run Recommended) -The following resources include the minimal runnable set and can be copied directly. +Firstly, create the ServiceAccount and ClusterRole to grant permissions to the CronJob. ```yaml --- @@ -155,11 +169,15 @@ kind: ServiceAccount metadata: name: ac-images-pruner-sa namespace: cpaas-system + labels: + cpaas.io/cleanup: ac-images-pruner --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: ac-images-pruner-read + name: ac-images-pruner-role + labels: + cpaas.io/cleanup: ac-images-pruner rules: - apiGroups: [""] resources: ["pods", "replicationcontrollers"] @@ -173,11 +191,16 @@ rules: - apiGroups: [""] resources: ["pods/exec"] verbs: ["create"] + - apiGroups: ["registry.alauda.io"] + resources: ["images"] + verbs: ["get", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: ac-images-pruner-read-binding + name: ac-images-pruner-rolebinding + labels: + cpaas.io/cleanup: ac-images-pruner subjects: - kind: ServiceAccount name: ac-images-pruner-sa @@ -185,27 +208,42 @@ subjects: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: ac-images-pruner-read + name: ac-images-pruner-role +``` + +Then, create the `CronJob`. + +(Dry-run template) This CronJob runs image pruning every 6 hours, keeps images created within the last 7 days and the latest 5 revisions per repository, and excludes repositories under `cpaas-system` namespace from cleanup. + +```yaml --- apiVersion: batch/v1 kind: CronJob metadata: name: ac-prune-images-cronjob namespace: cpaas-system + labels: + cpaas.io/cleanup: ac-images-pruner spec: schedule: "0 */6 * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: template: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: serviceAccountName: ac-images-pruner-sa restartPolicy: Never containers: - name: ac - image: /alauda/ac: + image: /alauda/ac: args: - adm - prune @@ -213,6 +251,10 @@ spec: - --keep-younger-than=168h - --keep-tag-revisions=5 - --whitelist=^cpaas-system/.* + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65532 ``` Apply resources: @@ -244,20 +286,28 @@ kind: CronJob metadata: name: ac-prune-images-daily namespace: cpaas-system + labels: + cpaas.io/cleanup: ac-images-pruner spec: schedule: "0 2 * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: template: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: - serviceAccountName: ac-tester-sa + serviceAccountName: ac-images-pruner-sa restartPolicy: Never containers: - name: ac - image: /alauda/ac: + image: /alauda/ac: args: - adm - prune @@ -265,6 +315,10 @@ spec: - --keep-younger-than=168h - --keep-tag-revisions=5 - --whitelist=^cpaas-system/.* + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65532 ``` ### Scenario 2: Weekly steady cleanup (recommended for production) @@ -275,20 +329,28 @@ kind: CronJob metadata: name: ac-prune-images-weekly namespace: cpaas-system + labels: + cpaas.io/cleanup: ac-images-pruner spec: schedule: "30 3 * * 0" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: template: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: - serviceAccountName: ac-tester-sa + serviceAccountName: ac-images-pruner-sa restartPolicy: Never containers: - name: ac - image: /alauda/ac: + image: /alauda/ac: args: - adm - prune @@ -297,6 +359,10 @@ spec: - --keep-tag-revisions=10 - --whitelist=^cpaas-system/.* - --confirm + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65532 ``` ### Scenario 3: Monthly aggressive cleanup (with whitelist protection) @@ -307,20 +373,28 @@ kind: CronJob metadata: name: ac-prune-images-monthly-aggressive namespace: cpaas-system + labels: + cpaas.io/cleanup: ac-images-pruner spec: schedule: "0 4 1 * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 2 failedJobsHistoryLimit: 5 jobTemplate: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: template: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: - serviceAccountName: ac-tester-sa + serviceAccountName: ac-images-pruner-sa restartPolicy: Never containers: - name: ac - image: /alauda/ac: + image: /alauda/ac: args: - adm - prune @@ -329,6 +403,10 @@ spec: - --whitelist=^cpaas-system/.* - --whitelist=^pro-ns1/base/.* - --confirm + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65532 ``` ### Scenario 4: Weekly cleanup with GC (off-peak window) @@ -339,20 +417,28 @@ kind: CronJob metadata: name: ac-prune-images-with-gc namespace: cpaas-system + labels: + cpaas.io/cleanup: ac-images-pruner spec: schedule: "0 1 * * 6" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 2 failedJobsHistoryLimit: 5 jobTemplate: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: template: + metadata: + labels: + cpaas.io/cleanup: ac-images-pruner spec: - serviceAccountName: ac-tester-sa + serviceAccountName: ac-images-pruner-sa restartPolicy: Never containers: - name: ac - image: /alauda/ac: + image: /alauda/ac: args: - adm - prune @@ -361,6 +447,10 @@ spec: - --keep-tag-revisions=5 - --prune-registry - --confirm + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65532 ``` ## Verification @@ -368,9 +458,9 @@ spec: Run these commands after deployment: ```bash -ac get cronjob -n cpaas-system -ac get job -n cpaas-system -ac get pod -n cpaas-system +ac get cronjob -n cpaas-system -l cpaas.io/cleanup=ac-images-pruner +ac get job -n cpaas-system -l cpaas.io/cleanup=ac-images-pruner +ac get pod -n cpaas-system -l cpaas.io/cleanup=ac-images-pruner ``` Expected result: @@ -396,7 +486,19 @@ Typical successful `Pod` logs (example): DRY RUN: Would delete pro-ns1/demo/bash:5 [4/5] Summary Candidates for pruning: 1 -[5/5] Dry run mode, skipping trigger registry garbage collection. +[5/5] Skipping registry garbage collection because --prune-registry is not set. +``` + +## Cleanup resources of the demo + +```bash +# Delete namespace-scoped resources (such as CronJob, Job, Pod, ServiceAccount, etc) +kubectl -n cpaas-system delete cronjob,job,pod,serviceaccount \ + -l cpaas.io/cleanup=ac-images-pruner --ignore-not-found + +# Delete cluster-scoped resources (such as ClusterRole, ClusterRoleBinding, etc) +kubectl delete clusterrole,clusterrolebinding \ + -l cpaas.io/cleanup=ac-images-pruner --ignore-not-found ``` ## Troubleshooting @@ -404,7 +506,8 @@ Typical successful `Pod` logs (example): | Symptom | Possible cause | Suggested action | |---------|----------------|------------------| | `no current context set` | Image does not include in-cluster fallback and no `ac login` session exists in container | Upgrade to a compatible AC version and verify SA token mount | -| `forbidden` / `cannot list ...` | ServiceAccount lacks RBAC for scan resources | Add required list/get/watch permissions to `ClusterRole` | +| `forbidden` / `cannot list ...` | ServiceAccount lacks RBAC for scan resources or registry image resource | Add required list/get/watch scan permissions and `registry.alauda.io/images` permissions (`get`, and `delete` if using `--confirm`) | +| `401` / `403` when fetching registry tags/manifests in Pod | In-cluster ServiceAccount token is not authorized by registry proxy | Verify proxy authn/authz config and grant required RBAC to the calling ServiceAccount | | `failed to list registry pods` | No access to registry Pod in `cpaas-system` | Verify RBAC and namespace settings | | Registry auth-related errors | Registry auth policy does not match current token mode | Verify `--registry-url` and registry token/anonymous policy | | No prune candidates | Retention settings are too strict or whitelist is too broad | Run Dry Run and tune `--keep-*` and `--whitelist` |