From e5eba1e38b9307f63db64a15b4551284c39cd9fd Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Wed, 1 Jul 2026 18:29:33 +1200 Subject: [PATCH 01/12] docs: update Velero docs for 1.17/1.18 Kopia support and Restic preservation - Document Velero 1.11.0-1.18.x compatibility, with Restic for 1.16 and earlier and Kopia for 1.17 and later - Update install instructions to use version-aware velero install flags - Add LVP migration guidance and Kopia troubleshooting sections - Rename _resticDaemonSet.mdx to _node-agent-daemonset.mdx - Add Kopia, Restic, MinIO, LVP, and related terms to Vale vocabularies - Add .tools to .gitignore --- .gitignore | 3 + docs/enterprise/snapshots-config-workflow.md | 4 +- .../snapshots-configuring-hostpath.md | 14 ++- docs/enterprise/snapshots-configuring-nfs.md | 14 ++- .../snapshots-storage-destinations.md | 39 ++++---- ...napshots-troubleshooting-backup-restore.md | 88 +++++++++++++++--- .../snapshots-velero-cli-installing.md | 18 ++-- .../snapshots-velero-installing-config.mdx | 2 +- .../enterprise/snapshots-velero-upgrading.mdx | 92 +++++++++++++++++++ docs/partials/snapshots/_installVelero.mdx | 19 +++- .../snapshots/_node-agent-daemonset.mdx | 7 ++ .../snapshots/_node-agent-mem-limit.mdx | 10 +- .../snapshots/_registryCredentialsNote.mdx | 2 +- docs/partials/snapshots/_resticDaemonSet.mdx | 7 -- .../snapshots/_velero-compatibility.mdx | 3 +- .../kots-cli-velero-configure-hostpath.mdx | 4 + .../kots-cli-velero-configure-nfs.mdx | 4 + docs/vendor/snapshots-overview.mdx | 10 +- sidebars.js | 3 +- .../config/vocabularies/TechJargon/accept.txt | 6 ++ .../ThirdPartyProducts/accept.txt | 7 ++ 21 files changed, 283 insertions(+), 73 deletions(-) create mode 100644 docs/enterprise/snapshots-velero-upgrading.mdx create mode 100644 docs/partials/snapshots/_node-agent-daemonset.mdx delete mode 100644 docs/partials/snapshots/_resticDaemonSet.mdx diff --git a/.gitignore b/.gitignore index d64b3dc3e5..0a8a1fe57d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ yarn-error.log* # Git worktrees .worktrees + +# Local tools / research directories +.tools diff --git a/docs/enterprise/snapshots-config-workflow.md b/docs/enterprise/snapshots-config-workflow.md index 8fcf31b7dc..a937263026 100644 --- a/docs/enterprise/snapshots-config-workflow.md +++ b/docs/enterprise/snapshots-config-workflow.md @@ -22,7 +22,7 @@ To configure snapshots for embedded clusters: - [Configure an NFS Storage Destination](snapshots-configuring-nfs) - [Configure Other Storage Destinations](snapshots-storage-destinations) -1. Optionally increase the default memory for the node-agent (restic) Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). +1. Optionally increase the default memory for the node-agent Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). ## Configuring backup storage for existing clusters @@ -38,7 +38,7 @@ Follow this process to install Velero and configure the snapshots feature: - [Configure an NFS Storage Destination](snapshots-configuring-nfs) - [Configure Other Storage Destinations](snapshots-storage-destinations) -1. Enable access to the Velero namespace if you are using RBAC and optionally increase the default memory for the node-agent (restic) Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). +1. Enable access to the Velero namespace if you are using RBAC and optionally increase the default memory for the node-agent Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). ## Next step diff --git a/docs/enterprise/snapshots-configuring-hostpath.md b/docs/enterprise/snapshots-configuring-hostpath.md index 7225b379cf..23516cc47f 100644 --- a/docs/enterprise/snapshots-configuring-hostpath.md +++ b/docs/enterprise/snapshots-configuring-hostpath.md @@ -1,6 +1,6 @@ import InstallVelero from "../partials/snapshots/_installVelero.mdx" import RegistryCredNote from "../partials/snapshots/_registryCredentialsNote.mdx" -import ResticDaemonSet from "../partials/snapshots/_resticDaemonSet.mdx" +import NodeAgentDaemonSet from "../partials/snapshots/_node-agent-daemonset.mdx" import UpdateDefaultStorage from "../partials/snapshots/_updateDefaultStorage.mdx" import CheckVersion from "../partials/snapshots/_checkVersion.mdx" import KotsAvailability from "../partials/kots/_kots-availability.mdx" @@ -15,6 +15,10 @@ This topic describes how to install Velero and configure a host path as your sto +:::important +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. If you use LVP for HostPath storage, migrate to a Kopia-compatible destination before upgrading to Velero 1.17 or later. Existing LVP backups from Velero 1.16 and earlier are not restorable on Velero 1.17 or later. For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). +::: + ## Requirements * The host path must be a dedicated directory. Do not use a partition used by a service like Docker or Kubernetes for ephemeral storage. @@ -37,7 +41,7 @@ To install Velero and configure host path storage in online environments: 1. -1. +1. 1. Run the following command to configure the host path storage destination: @@ -51,9 +55,9 @@ To install Velero and configure host path storage in online environments: For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. -## Install Velero and configure host path storage in air gapped environments +## Install Velero and configure host path storage in air-gapped environments -To install Velero and configure host path storage in air gapped environments: +To install Velero and configure host path storage in air-gapped environments: 1. @@ -61,7 +65,7 @@ To install Velero and configure host path storage in air gapped environments: -1. +1. 1. Run the following command to configure the host path storage destination: diff --git a/docs/enterprise/snapshots-configuring-nfs.md b/docs/enterprise/snapshots-configuring-nfs.md index 1babc6622d..1cc0b82d33 100644 --- a/docs/enterprise/snapshots-configuring-nfs.md +++ b/docs/enterprise/snapshots-configuring-nfs.md @@ -1,6 +1,6 @@ import InstallVelero from "../partials/snapshots/_installVelero.mdx" import RegistryCredNote from "../partials/snapshots/_registryCredentialsNote.mdx" -import ResticDaemonSet from "../partials/snapshots/_resticDaemonSet.mdx" +import NodeAgentDaemonSet from "../partials/snapshots/_node-agent-daemonset.mdx" import UpdateDefaultStorage from "../partials/snapshots/_updateDefaultStorage.mdx" import CheckVersion from "../partials/snapshots/_checkVersion.mdx" import KotsAvailability from "../partials/kots/_kots-availability.mdx" @@ -15,6 +15,10 @@ This topic describes how to install Velero and configure a Network File System ( +:::important +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. If you use LVP for NFS storage, migrate to a Kopia-compatible destination before upgrading to Velero 1.17 or later. Existing LVP backups from Velero 1.16 and earlier are not restorable on Velero 1.17 or later. For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). +::: + ## Requirements Configuring an NFS server as a snapshots storage destination has the following requirements: @@ -38,7 +42,7 @@ To install Velero and configure NFS storage in an online environment: 1. -1. +1. 1. Run the following command to configure the NFS storage destination: @@ -53,9 +57,9 @@ To install Velero and configure NFS storage in an online environment: For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. -## Install Velero and configure NFS storage in air gapped environments +## Install Velero and configure NFS storage in air-gapped environments -To install Velero and configure NFS storage in air gapped environments: +To install Velero and configure NFS storage in air-gapped environments: 1. @@ -63,7 +67,7 @@ To install Velero and configure NFS storage in air gapped environments: -1. +1. 1. Run the following command to configure the NFS storage destination: diff --git a/docs/enterprise/snapshots-storage-destinations.md b/docs/enterprise/snapshots-storage-destinations.md index 18a9613f9f..d4046225da 100644 --- a/docs/enterprise/snapshots-storage-destinations.md +++ b/docs/enterprise/snapshots-storage-destinations.md @@ -34,20 +34,21 @@ To install Velero and configure an AWS storage destination: 1. Run the `velero install` command with these additional flags: - * **Velero 1.10 and later**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. + * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. + * **Velero 1.10 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. **Example:** ``` velero install \ - --provider aws \ - --plugins velero/velero-plugin-for-aws:v1.2.0 \ - --bucket $BUCKET \ - --backup-location-config region=$REGION \ - --secret-file CREDS_FILE \ - --use-node-agent --uploader-type=restic \ - --use-volume-snapshots=false + --provider aws \ + --plugins velero/velero-plugin-for-aws:v1.14.2 \ + --bucket $BUCKET \ + --backup-location-config region=$REGION \ + --secret-file CREDS_FILE \ + --use-node-agent \ + --use-volume-snapshots=false ``` ## Configure GCP storage for online environments @@ -59,7 +60,8 @@ To install Velero and configure a GCP storage destination: 1. Follow the instructions for installing Velero on GCP in the [velero-plugin-for-gcp](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) repository in GitHub. 1. Run the `velero install` command with these additional flags: - * **Velero 1.10 and later**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. + * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. + * **Velero 1.10 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. **Example:** @@ -67,10 +69,10 @@ To install Velero and configure a GCP storage destination: ``` velero install \ --provider gcp \ - --plugins velero/velero-plugin-for-gcp:v1.5.0 \ + --plugins velero/velero-plugin-for-gcp:v1.14.2 \ --bucket $BUCKET \ - --secret-file ./CREDS_FILE - --use-node-agent --uploader-type=restic \ + --secret-file ./CREDS_FILE \ + --use-node-agent \ --use-volume-snapshots=false ``` @@ -83,7 +85,8 @@ To install Velero and configure an Azure storage destination: 1. Follow the instructions for [Install Velero on Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) in the Velero documentation. 1. Run the `velero install` command with these additional flags: - * **Velero 1.10 and later**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. + * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. + * **Velero 1.10 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. **Example:** @@ -91,12 +94,12 @@ To install Velero and configure an Azure storage destination: ``` velero install \ --provider azure \ - --plugins velero/velero-plugin-for-microsoft-azure:v1.5.0 \ + --plugins velero/velero-plugin-for-microsoft-azure:v1.14.2 \ --bucket $BLOB_CONTAINER \ --secret-file ./CREDS_FILE \ --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ - --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] - --use-node-agent --uploader-type=restic \ + --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ + --use-node-agent \ --use-volume-snapshots=false ``` @@ -142,7 +145,7 @@ kubectl kots velero configure-other-s3 \ If no Velero installation is detected, instructions are displayed to install Velero and configure the storage destination. -## Configure s3-compatible storage for air gapped environments +## Configure s3-compatible storage for air-gapped environments > Introduced in Replicated KOTS v1.94.0 @@ -151,7 +154,7 @@ The following S3-compatible object stores are supported for storing backups with - Ceph RADOS v12.2.7 - MinIO -Run the following command to install Velero and configure an S3-compatible storage destination in an air gapped environment. For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. +Run the following command to install Velero and configure an S3-compatible storage destination in an air-gapped environment. For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. ```bash kubectl kots velero configure-other-s3 \ diff --git a/docs/enterprise/snapshots-troubleshooting-backup-restore.md b/docs/enterprise/snapshots-troubleshooting-backup-restore.md index 28674dc0d5..cf6d462dab 100644 --- a/docs/enterprise/snapshots-troubleshooting-backup-restore.md +++ b/docs/enterprise/snapshots-troubleshooting-backup-restore.md @@ -99,27 +99,29 @@ timed out after 12h0m0s #### Cause -This error message appears when the node-agent (restic) Pod operation timeout limit is reached. In Velero v1.4.2 and later, the default timeout is 240 minutes. +This error message appears when the node-agent Pod operation timeout limit is reached. In Velero v1.4.2 and later, the default timeout is 240 minutes. -Restic is an open-source backup tool. Velero integrates with Restic to provide a solution for backing up and restoring Kubernetes volumes. For more information about the Velero Restic integration, see [File System Backup](https://velero.io/docs/v1.10/file-system-backup/) in the Velero documentation. +For Velero 1.16 and earlier, Velero integrates with Restic to provide a solution for backing up and restoring Kubernetes volumes. For more information, see [File System Backup](https://velero.io/docs/v1.10/file-system-backup/) in the Velero documentation. + +For Velero 1.17 and later, Velero uses Kopia for file-system backups by default. #### Solution Use the kubectl Kubernetes command-line tool to patch the Velero deployment to increase the timeout: -**Velero version 1.10 and later**: +**Velero 1.17 and later**: ```bash kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--fs-backup-timeout=TIMEOUT_LIMIT"}]' ``` -**Velero versions less than 1.10**: +**Velero 1.16 and earlier**: ```bash kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--restic-timeout=TIMEOUT_LIMIT"}]' ``` -Replace `TIMEOUT_LIMIT` with a length of time for the node-agent (restic) Pod operation timeout in hours, minutes, and seconds. Use the format `0h0m0s`. For example, `48h30m0s`. +Replace `TIMEOUT_LIMIT` with a length of time for the node-agent Pod operation timeout in hours, minutes, and seconds. Use the format `0h0m0s`. For example, `48h30m0s`. :::note The timeout value reverts back to the default value if you rerun the `velero install` command. @@ -129,7 +131,7 @@ The timeout value reverts back to the default value if you rerun the `velero ins #### Symptom -The node-agent (restic) Pod is killed by the Linux kernel Out Of Memory (OOM) killer or snapshots are failing with errors simlar to: +The node-agent Pod is killed by the Linux kernel Out Of Memory (OOM) killer or snapshots are failing with errors similar to: ``` pod volume backup failed: ... signal: killed @@ -137,9 +139,9 @@ pod volume backup failed: ... signal: killed #### Cause -Velero sets default limits for the velero Pod and the node-agent (restic) Pod during installation. There is a known issue with Restic that causes high memory usage, which can result in failures during snapshot creation when the Pod reaches the memory limit. +Velero sets default limits for the velero Pod and the node-agent Pod during installation. For Velero 1.16 and earlier, there is a known issue with Restic that causes high memory usage, which can result in failures during snapshot creation when the Pod reaches the memory limit. For Velero 1.17 and later, Velero uses Kopia for file-system backups by default, and large volumes can also require a higher memory limit. -For more information, see the [Restic backup — OOM-killed on raspberry pi after backing up another computer to same repo](https://github.com/restic/restic/issues/1988) issue in the restic GitHub repository. +For more information about the Restic issue, see the [Restic backup — OOM-killed on raspberry pi after backing up another computer to same repo](https://github.com/restic/restic/issues/1988) issue in the Restic GitHub repository. #### Solution @@ -157,7 +159,7 @@ Error backing up item...Warning: at least one source file could not be read #### Cause -There are file changes between Restic's initial scan of the volume and during the backup to Restic store. +For Velero 1.16 and earlier, there are file changes between Restic's initial scan of the volume and during the backup to the Restic store. #### Solution @@ -167,9 +169,73 @@ To resolve this issue, do one of the following: * Freeze the file system to ensure all pending disk I/O operations have completed prior to taking a snapshot. For more information, see [Hook Example with fsfreeze](https://velero.io/docs/main/backup-hooks/#hook-example-with-fsfreeze) in the Velero documentation. +## Kopia file-system backup issues (Velero 1.17 and later) + +### Data mover pods do not start or complete + +#### Cause + +For Velero 1.17 and later, Kopia spawns data mover pods from the node-agent. If a backup or restore stays in progress, the data mover pods might not start or complete. + +#### Solution + +Check the node-agent logs for errors that prevent data mover pods from starting. Verify that the data mover pod image can be pulled and that any pod security policies or security context constraints allow the pod to start. + +### BackupRepository is not available + +#### Cause + +For Velero 1.17 and later, Kopia uses `BackupRepository` custom resources (CRs) to manage repositories. A backup or restore can fail if the `BackupRepository` CR is not available or is in a failed state. + +#### Solution + +Check the status of the `BackupRepository` CRs: + +```bash +kubectl get backuprepositories -n velero +``` + +Describe any CR that is not in a Ready state to view the error: + +```bash +kubectl describe backuprepository BACKUP_REPOSITORY_NAME -n velero +``` + +Common errors include invalid credentials, network connectivity issues, or problems with the underlying storage. After you resolve the issue, Velero retries the backup repository operations. + +### Read-only root filesystem errors + +#### Cause + +For Velero 1.17 and later, Kopia needs writable directories for cache and configuration. The default paths are `/home/cnb/udmrepo` and `/home/cnb/.cache`. If `ReadOnlyRootFilesystem` is set for the Velero or node-agent pods, Kopia cannot write to these directories and the backup or restore fails. + +#### Solution + +Add `emptyDir` volumes for `/home/cnb/udmrepo` and `/home/cnb/.cache` to the Velero deployment and the node-agent daemon set. Mount the volumes at the required paths so Kopia can write cache and configuration data. + +### LVP storage location is unavailable after upgrade + +#### Cause + +The Local Volume Provider (LVP) is not compatible with Kopia. If you upgrade to Velero 1.17 or later and the existing storage location uses LVP, snapshots fail because Kopia cannot write to LVP storage. + +:::important +LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Before you upgrade, migrate to a Kopia-compatible storage destination. For more information, see [Upgrade Velero for Snapshots](snapshots-velero-upgrading). +::: + +#### Solution + +Before you upgrade to Velero 1.17 or later, migrate from LVP to a Kopia-compatible destination. Replicated recommends one of the following options: + +* Reinstall KOTS with `--with-minio=true`. +* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Ensure that the target Velero plugin is installed before you reconfigure the storage location. + +For more information, see [Upgrade Velero for Snapshots](snapshots-velero-upgrading). + + ## Snapshot restore is failing -### Service nodeport is already allocated +### Service NodePort is already allocated #### Symptom @@ -187,7 +253,7 @@ There is a known issue in Kubernetes versions earlier than version 1.19 where us This issue is fixed in Kubernetes version 1.19. To resolve this issue, upgrade to Kubernetes version 1.19 or later. -For more infromation about the fix, see https://github.com/kubernetes/kubernetes/pull/89937. +For more information about the fix, see https://github.com/kubernetes/kubernetes/pull/89937. ### Partial snapshot restore is stuck in progress diff --git a/docs/enterprise/snapshots-velero-cli-installing.md b/docs/enterprise/snapshots-velero-cli-installing.md index 891a917341..312aeafd38 100644 --- a/docs/enterprise/snapshots-velero-cli-installing.md +++ b/docs/enterprise/snapshots-velero-cli-installing.md @@ -10,6 +10,10 @@ For embedded clusters created with Replicated kURL, if the kURL Installer spec i +:::note +Velero 1.17 and later uses Kopia as the default file-system backup uploader. For more information, see [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). +::: + ## Install the Velero CLI in an online cluster To install the Velero CLI in an online cluster: @@ -34,10 +38,10 @@ To install the Velero CLI in an online cluster: **Example:** ``` - curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.10.1/velero-v1.10.1-linux-amd64.tar.gz + curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.18.2/velero-v1.18.2-linux-amd64.tar.gz ``` -1. Run the following command to uncompress the TAR file: +1. Run the following command to extract the TAR file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz @@ -55,9 +59,9 @@ To install the Velero CLI in an online cluster: You might get an error message stating that there are no matches for the server version. This is acceptable, as long as you get a confirmation for the client version. After the Velero installation, you also see the server version. -## Install the Velero CLI in an air gapped cluster +## Install the Velero CLI in an air-gapped cluster -To install the Velero CLI in an air gapped cluster: +To install the Velero CLI in an air-gapped cluster: 1. From a computer with internet access, check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although earlier versions of Velero are supported, Replicated recommends using the latest supported version. See [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). @@ -74,12 +78,12 @@ To install the Velero CLI in an air gapped cluster: **Example:** ``` - curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.10.1/velero-v1.10.1-linux-amd64.tar.gz + curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.18.2/velero-v1.18.2-linux-amd64.tar.gz ``` -1. Copy the TAR file to the air gapped node. +1. Copy the TAR file to the air-gapped node. -1. Run the following command to uncompress the TAR file: +1. Run the following command to extract the TAR file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz diff --git a/docs/enterprise/snapshots-velero-installing-config.mdx b/docs/enterprise/snapshots-velero-installing-config.mdx index f8fa92925a..e0cee057c3 100644 --- a/docs/enterprise/snapshots-velero-installing-config.mdx +++ b/docs/enterprise/snapshots-velero-installing-config.mdx @@ -34,7 +34,7 @@ For more information, see [`velero ensure-permissions`](/reference/kots-cli-vele This section applies to all online and air gap installations. -Velero sets default limits for the velero Pod and the node-agent (restic) Pod during installation. There is a known issue with restic that causes high memory usage, which can result in failures during backup creation when the Pod reaches the memory limit. +Velero sets default limits for the velero Pod and the node-agent Pod during installation. There is a known issue with the file-system backup uploader that causes high memory usage, which can result in failures during backup creation when the Pod reaches the memory limit. diff --git a/docs/enterprise/snapshots-velero-upgrading.mdx b/docs/enterprise/snapshots-velero-upgrading.mdx new file mode 100644 index 0000000000..54f3a01733 --- /dev/null +++ b/docs/enterprise/snapshots-velero-upgrading.mdx @@ -0,0 +1,92 @@ +import KotsAvailability from "../partials/kots/_kots-availability.mdx" + +# Upgrade Velero for snapshots + +This topic describes how to upgrade Velero for Replicated snapshots. + + + +## Overview + +Velero 1.17 and later uses Kopia as the default uploader for file-system backups. If you upgrade from Velero 1.16 or earlier, verify that your storage destination is compatible with Kopia before you upgrade. Some storage destinations, such as the Local Volume Provider (LVP), are not compatible with Kopia and must be migrated before the upgrade. + +For more information about snapshot troubleshooting, see [Troubleshoot Snapshots](snapshots-troubleshooting-backup-restore). + +## Before you upgrade + +Complete the following prerequisites before you upgrade Velero: + +1. Check the current storage destination type. If the storage location uses LVP, migrate to a Kopia-compatible destination before you upgrade. For more information, see [Migrate from Local Volume Provider (LVP) to a Kopia-compatible destination](#migrate-from-local-volume-provider-lvp-to-a-kopia-compatible-destination). + +1. Take a snapshot of the application before you upgrade. This snapshot lets you restore the application if the upgrade fails. + +1. Ensure that the Velero plugin for the target storage destination is installed. For example, verify that the plugin image is available in the registry for air-gapped environments. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). + +## Migrate from local volume provider (LVP) to a Kopia-compatible destination + +LVP is not compatible with Kopia. If Velero is configured to use LVP, you must migrate to a Kopia-compatible destination before you upgrade to Velero 1.17 or later. + +:::important +LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Replicated recommends that you migrate to a Kopia-compatible destination before you upgrade. +::: + +In-place reconfiguration from LVP to a filesystem-based Minio deployment is not supported. The `minio-enabled-snapshots=false` setting persists, so reconfiguring to filesystem Minio in the Admin Console does not enable a Kopia-compatible object store. + +Replicated recommends one of the following options: + +* Reinstall KOTS with `--with-minio=true`. This installs a Minio object store that is compatible with Kopia. +* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Ensure that the target Velero plugin is installed before you reconfigure the storage location. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). + +For more information about LVP storage issues after upgrade, see [LVP storage location is unavailable after upgrade](snapshots-troubleshooting-backup-restore#lvp-storage-location-is-unavailable-after-upgrade) in _Troubleshoot Snapshots_. + +## Upgrade one version at a time + +Replicated recommends that you upgrade Velero one version at a time. For example, if you are running Velero 1.15, upgrade to Velero 1.16 first, verify that snapshots work, and then upgrade to Velero 1.17. Upgrading incrementally reduces the risk of incompatible custom resources or storage locations. + +To upgrade Velero, follow the upgrade instructions for the target Velero version in the [Velero documentation](https://velero.io/docs/). KOTS does not provide an Admin Console button or a KOTS CLI command to upgrade Velero. The cluster administrator performs the upgrade manually. + +## Verify the upgrade + +After you upgrade Velero, verify that the upgrade is successful: + +1. Run `velero version` to confirm that the expected Velero version is installed. + +1. Check the status of the `BackupStorageLocation`: + + ```bash + velero backup-location get + ``` + + The BackupStorageLocation must be in an `Available` state before you create backups. If it is not available, describe the BackupStorageLocation to view the error: + + ```bash + kubectl describe backupstoragelocation -n velero + ``` + +1. Check the status of the `BackupRepository` CRs: + + ```bash + kubectl get backuprepositories -n velero + ``` + + All `BackupRepository` CRs must be in a `Ready` state before you create backups. If a CR is not ready, describe the CR to view the error: + + ```bash + kubectl describe backuprepository -n velero + ``` + +1. Create a test backup and verify that it completes successfully. For more information, see [Create and Schedule Backups](snapshots-creating). + +## Restore existing Restic backups + +Restic backups created on Velero 1.16 and earlier can be restored on Velero 1.17 and 1.18. Velero 1.17 and 1.18 retain the ability to restore Restic backups. To restore an existing Restic backup, use the same storage location and credentials that were used to create the backup. + +## Additional resources + +* [Troubleshoot Snapshots](snapshots-troubleshooting-backup-restore) +* [Configure Other Storage Destinations](snapshots-storage-destinations) +* [Configure a Host Path Storage Destination](snapshots-configuring-hostpath) +* [Configure an NFS Storage Destination](snapshots-configuring-nfs) +* [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config) +* [Create and Schedule Backups](snapshots-creating) +* [Update storage settings](snapshots-updating-with-admin-console) diff --git a/docs/partials/snapshots/_installVelero.mdx b/docs/partials/snapshots/_installVelero.mdx index cdd03f7aa4..a5fa516433 100644 --- a/docs/partials/snapshots/_installVelero.mdx +++ b/docs/partials/snapshots/_installVelero.mdx @@ -1,6 +1,17 @@ Run one of the following commands to install Velero, depending on the version of the velero CLI you are using: - * **Velero v1.10 and later**: + * **Velero v1.17 and later**: + + ```bash + velero install \ + --no-default-backup-location \ + --no-secret \ + --use-node-agent \ + --use-volume-snapshots=false \ + --plugins velero/velero-plugin-for-aws:v1.14.2 + ``` + + * **Velero v1.10 to v1.16**: ```bash velero install \ @@ -8,7 +19,7 @@ Run one of the following commands to install Velero, depending on the version of --no-secret \ --use-node-agent --uploader-type=restic \ --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.5.3 + --plugins velero/velero-plugin-for-aws:v1.14.2 ``` * **Velero versions earlier than v1.10**: @@ -19,5 +30,5 @@ Run one of the following commands to install Velero, depending on the version of --no-secret \ --use-restic \ --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.5.3 - ``` \ No newline at end of file + --plugins velero/velero-plugin-for-aws:v1.14.2 + ``` diff --git a/docs/partials/snapshots/_node-agent-daemonset.mdx b/docs/partials/snapshots/_node-agent-daemonset.mdx new file mode 100644 index 0000000000..236d375f88 --- /dev/null +++ b/docs/partials/snapshots/_node-agent-daemonset.mdx @@ -0,0 +1,7 @@ +Configure the node-agent DaemonSet specification if your cluster uses one of the following Kubernetes distributions: + * RancherOS + * OpenShift + * Microsoft Azure + * VMware Tanzu Kubernetes Grid Integrated Edition + +For information about how to configure the node-agent DaemonSet for these distributions, see [Configure node-agent DaemonSet spec](https://velero.io/docs/v1.17/file-system-backup/#configure-node-agent-daemonset-spec) in the Velero documentation. \ No newline at end of file diff --git a/docs/partials/snapshots/_node-agent-mem-limit.mdx b/docs/partials/snapshots/_node-agent-mem-limit.mdx index 1f66e903b4..43fa521ef4 100644 --- a/docs/partials/snapshots/_node-agent-mem-limit.mdx +++ b/docs/partials/snapshots/_node-agent-mem-limit.mdx @@ -1,8 +1,8 @@ -Increase the default memory limit for the node-agent (restic) Pod if your application is particularly large. For more information about configuring Velero resource requests and limits, see [Customize resource requests and limits](https://velero.io/docs/v1.10/customize-installation/#customize-resource-requests-and-limits) in the Velero documentation. +Increase the default memory limit for the node-agent Pod if your application is particularly large. For more information about configuring Velero resource requests and limits, see [Customize resource requests and limits](https://velero.io/docs/v1.17/customize-installation/#customize-resource-requests-and-limits) in the Velero documentation. -For example, the following kubectl commands will increase the memory limit for the node-agent (restic) daemon set from the default of 1Gi to 2Gi. +For example, the following kubectl commands will increase the memory limit for the node-agent DaemonSet from the default of 1Gi to 2Gi. -**Velero 1.10 and later**: +**Velero 1.10 and later**: ``` kubectl -n velero patch daemonset node-agent -p '{"spec":{"template":{"spec":{"containers":[{"name":"node-agent","resources":{"limits":{"memory":"2Gi"}}}]}}}}' @@ -14,9 +14,9 @@ kubectl -n velero patch daemonset node-agent -p '{"spec":{"template":{"spec":{"c kubectl -n velero patch daemonset restic -p '{"spec":{"template":{"spec":{"containers":[{"name":"restic","resources":{"limits":{"memory":"2Gi"}}}]}}}}' ``` -Alternatively, you can potentially avoid the node-agent (restic) Pod reaching the memory limit during snapshot creation by running the following kubectl command to lower the memory garbage collection target percentage on the node-agent (restic) daemon set: +Alternatively, you can potentially avoid the node-agent Pod reaching the memory limit during snapshot creation by running the following kubectl command to lower the memory garbage collection target percentage on the node-agent DaemonSet: -**Velero 1.10 and later**: +**Velero 1.10 and later**: ``` kubectl -n velero set env daemonset/node-agent GOGC=1 diff --git a/docs/partials/snapshots/_registryCredentialsNote.mdx b/docs/partials/snapshots/_registryCredentialsNote.mdx index fbab6aaeda..bf29501df2 100644 --- a/docs/partials/snapshots/_registryCredentialsNote.mdx +++ b/docs/partials/snapshots/_registryCredentialsNote.mdx @@ -1,5 +1,5 @@ :::note -It is typical for the velero and node-agent (restic) Pods to be in the `ErrImagePull` or `ImagePullBackOff` state after you run the `velero install` command because Velero does not support passing registry credentials during installation. In Replicated KOTS v1.94.0 and later, this situation resolves itself after you complete the instructions to configure the storage destination. +It is typical for the velero and node-agent Pods to be in the `ErrImagePull` or `ImagePullBackOff` state after you run the `velero install` command because Velero does not support passing registry credentials during installation. In Replicated KOTS v1.94.0 and later, this situation resolves itself after you complete the instructions to configure the storage destination. If you are on an earlier version of KOTS, Replicated recommends that you upgrade to KOTS v1.94.0 or later. Otherwise, you must patch the Velero deployment manually and add the image pull secret to access the registry. ::: \ No newline at end of file diff --git a/docs/partials/snapshots/_resticDaemonSet.mdx b/docs/partials/snapshots/_resticDaemonSet.mdx deleted file mode 100644 index b54778e92b..0000000000 --- a/docs/partials/snapshots/_resticDaemonSet.mdx +++ /dev/null @@ -1,7 +0,0 @@ -Configure the Restic DaemonSet specification if your cluster uses one of the following Kubernetes distributions: - * RancherOS - * OpenShift - * Microsoft Azure - * VMware Tanzu Kubernetes Grid Integrated Edition - -For information about how to configure the Restic DaemonSet for these distributions, see [Configure Restic DaemonSet spec](https://velero.io/docs/v1.9/restic/#configure-restic-daemonset-spec) in the Velero documentation. \ No newline at end of file diff --git a/docs/partials/snapshots/_velero-compatibility.mdx b/docs/partials/snapshots/_velero-compatibility.mdx index c546279f61..97bcdb4eed 100644 --- a/docs/partials/snapshots/_velero-compatibility.mdx +++ b/docs/partials/snapshots/_velero-compatibility.mdx @@ -1,10 +1,11 @@ The following table lists which versions of Velero are compatible with each version of KOTS. For a list of the supported Kubernetes versions for each Velero version, see [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) in the vmware-tanzu/velero repository. -:::note If you're running KOTS with Velero 1.17+, pod volume backups using Restic are unavailable. This is because Velero 1.17 removed the `--uploader-type=restic` flag. Restores of existing Restic backups still work. If you depend on snapshots, stay on Velero 1.16.x until a KOTS update with Kopia support ships. +:::note KOTS supports Velero 1.11.0 through 1.18.x. For Velero 1.16 and earlier, KOTS uses Restic to back up pod volume data. For Velero 1.17 and later, KOTS uses Kopia. Existing Restic backups are restorable on Velero 1.17 and 1.18. Velero 1.19 is not supported. ::: | KOTS version | Velero version | |------|-------------| +| TBD and later | 1.11.0 to 1.18.x | | 1.125.2 and later | 1.11.0 to 1.16.2 | | 1.125.0 and later | 1.11.0 to 1.16.1 | | 1.124.18 and later | 1.11.0 to 1.16.0 | diff --git a/docs/reference/kots-cli-velero-configure-hostpath.mdx b/docs/reference/kots-cli-velero-configure-hostpath.mdx index 97d4b76d53..2f545fec0c 100644 --- a/docs/reference/kots-cli-velero-configure-hostpath.mdx +++ b/docs/reference/kots-cli-velero-configure-hostpath.mdx @@ -8,6 +8,10 @@ import Help from "../partials/kots-cli/_help.mdx" Configure snapshots to use a host path as storage destination. +:::note +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. By default, KOTS uses the S3-compatible filesystem MinIO path for HostPath storage when MinIO is enabled. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. For more information, see [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading). +::: + ### Usage ```bash diff --git a/docs/reference/kots-cli-velero-configure-nfs.mdx b/docs/reference/kots-cli-velero-configure-nfs.mdx index 11405c3ddd..1616226d4e 100644 --- a/docs/reference/kots-cli-velero-configure-nfs.mdx +++ b/docs/reference/kots-cli-velero-configure-nfs.mdx @@ -8,6 +8,10 @@ import Help from "../partials/kots-cli/_help.mdx" Configures snapshots to use NFS as storage destination. +:::note +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. By default, KOTS uses the S3-compatible filesystem MinIO path for NFS storage when MinIO is enabled. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. For more information, see [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading). +::: + ### Usage ```bash diff --git a/docs/vendor/snapshots-overview.mdx b/docs/vendor/snapshots-overview.mdx index 8810a65377..95f335a40a 100644 --- a/docs/vendor/snapshots-overview.mdx +++ b/docs/vendor/snapshots-overview.mdx @@ -60,7 +60,7 @@ This section describes the types of backups that are supported with snapshots. F Full backups (also referred to as _instance_ backups) include the KOTS Admin Console and all application data, including application volumes and manifest files. -For clusters created with Replicated kURL, full backups also back up the Docker registry, which is required for air gapped installations. +For clusters created with Replicated kURL, full backups also back up the Docker registry, which is required for air-gapped installations. If you manage multiple applications with the Admin Console, data from all applications that support backups is included in a full backup. To be included in full backups, each application must include a manifest file with `kind: Backup` and `apiVersion: velero.io/v1`, which you can check for in the Admin Console. @@ -76,9 +76,9 @@ Partial backups can be useful if you need to roll back after deploying a new app For disaster recovery, backups should be configured to use a storage destination that exists outside of the cluster. This is especially true for installations in clusters created with Replicated kURL, because the default storage location on these clusters is internal. -You can use a storage provider that is compatible with Velero as the storage destination for backups created with the Replicated snapshots feature. For a list of the compatible storage providers, see [Providers](https://velero.io/docs/v1.9/supported-providers/) in the Velero documentation. +You can use a storage provider that is compatible with Velero as the storage destination for backups created with the Replicated snapshots feature. For a list of the compatible storage providers, see [Providers](https://velero.io/docs/v1.17/supported-providers/) in the Velero documentation. -You initially configure backups on a supported storage provider backend using the KOTS CLI. If you want to change the storage destination after the initial configuration, you can use the the **Snapshots** page in the Admin Console, which has built-in support for the following storage destinations: +You initially configure backups on a supported storage provider backend using the KOTS CLI. If you want to change the storage destination after the initial configuration, you can use the **Snapshots** page in the Admin Console, which has built-in support for the following storage destinations: - Amazon Web Services (AWS) - Google Cloud Provider (GCP) @@ -93,7 +93,7 @@ For information about how to configure backup storage destinations for snapshots ### What data is backed up? -Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, this also backs up the Docker registry, which is required for air gapped installations. +Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, this also backs up the Docker registry, which is required for air-gapped installations. #### Other object-stored data @@ -103,7 +103,7 @@ For object-stored data that is not KOTS-specific and does not use persistentVolu #### Pod volume data -Replicated supports only the restic backup program for pod volume data. +KOTS uses Velero's file-system backup uploader to back up pod volume data. For Velero 1.16 and earlier, the uploader is Restic. For Velero 1.17 and later, the uploader is Kopia. By default, Velero requires that you opt-in to have pod volumes backed up. In the Backup resource that you configure to enable snapshots, you must annotate each specific volume that you want to back up. For more information about including and excluding pod volumes, see [Configure Snapshots](/vendor/snapshots-configuring-backups). diff --git a/sidebars.js b/sidebars.js index f32c9d0225..d6fef8284c 100644 --- a/sidebars.js +++ b/sidebars.js @@ -395,7 +395,7 @@ const sidebars = { }, { type: "category", - label: "Configure Backup Storage for Snaphots", + label: "Configure Backup Storage for Snapshots", items: [ "enterprise/snapshots-velero-cli-installing", "enterprise/snapshots-configuring-hostpath", @@ -408,6 +408,7 @@ const sidebars = { "enterprise/snapshots-restoring-full", "enterprise/snapshots-updating-with-admin-console", "enterprise/snapshots-troubleshooting-backup-restore", + "enterprise/snapshots-velero-upgrading", ], }, { diff --git a/styles/config/vocabularies/TechJargon/accept.txt b/styles/config/vocabularies/TechJargon/accept.txt index b8e9634d81..71eb85d5c1 100644 --- a/styles/config/vocabularies/TechJargon/accept.txt +++ b/styles/config/vocabularies/TechJargon/accept.txt @@ -124,3 +124,9 @@ discoverability idempotently monorepo Monorepo +LVP +uploader +CRs +BackupRepository +preconfigured +crashloop diff --git a/styles/config/vocabularies/ThirdPartyProducts/accept.txt b/styles/config/vocabularies/ThirdPartyProducts/accept.txt index 755fe64a59..0f0846b13e 100644 --- a/styles/config/vocabularies/ThirdPartyProducts/accept.txt +++ b/styles/config/vocabularies/ThirdPartyProducts/accept.txt @@ -49,3 +49,10 @@ Fulcio Rekor SecureBuild Sigstore +Kopia +Restic +MinIO +Minio +Ceph +VMware +Tanzu From 6008207640eb5393b5abd0a44b56584386da942f Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Wed, 1 Jul 2026 18:33:35 +1200 Subject: [PATCH 02/12] docs: add supported Kubernetes versions to Velero compatibility table Add a third column to the Velero compatibility table listing the Kubernetes versions tested for the latest Velero version in each supported range. Data sourced from the Velero Compatibility Matrix at https://github.com/vmware-tanzu/velero#velero-compatibility-matrix --- .../snapshots/_velero-compatibility.mdx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/partials/snapshots/_velero-compatibility.mdx b/docs/partials/snapshots/_velero-compatibility.mdx index 97bcdb4eed..98d7578881 100644 --- a/docs/partials/snapshots/_velero-compatibility.mdx +++ b/docs/partials/snapshots/_velero-compatibility.mdx @@ -1,14 +1,14 @@ -The following table lists which versions of Velero are compatible with each version of KOTS. For a list of the supported Kubernetes versions for each Velero version, see [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) in the vmware-tanzu/velero repository. +The following table lists the versions of Velero that are compatible with each version of KOTS, and the Kubernetes versions that each Velero version supports. For the full Velero Compatibility Matrix, see [Velero Compatibility Matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) in the vmware-tanzu/velero repository. :::note KOTS supports Velero 1.11.0 through 1.18.x. For Velero 1.16 and earlier, KOTS uses Restic to back up pod volume data. For Velero 1.17 and later, KOTS uses Kopia. Existing Restic backups are restorable on Velero 1.17 and 1.18. Velero 1.19 is not supported. ::: -| KOTS version | Velero version | -|------|-------------| -| TBD and later | 1.11.0 to 1.18.x | -| 1.125.2 and later | 1.11.0 to 1.16.2 | -| 1.125.0 and later | 1.11.0 to 1.16.1 | -| 1.124.18 and later | 1.11.0 to 1.16.0 | -| 1.124.0 and later | 1.11.0 to 1.15.2 | -| 1.123.1 and later | 1.11.0 to 1.15.1 | -| 1.117.3 and later | 1.11.0 to 1.14.1 | +| KOTS version | Velero version | Supported Kubernetes versions | +|------|-------------|-------------| +| TBD and later | 1.11.0 to 1.18.x | 1.33.7, 1.34.1, and 1.35.0 (Velero 1.18) | +| 1.125.2 and later | 1.11.0 to 1.16.2 | 1.31.4, 1.32.3, and 1.33.0 (Velero 1.16) | +| 1.125.0 and later | 1.11.0 to 1.16.1 | 1.31.4, 1.32.3, and 1.33.0 (Velero 1.16) | +| 1.124.18 and later | 1.11.0 to 1.16.0 | 1.31.4, 1.32.3, and 1.33.0 (Velero 1.16) | +| 1.124.0 and later | 1.11.0 to 1.15.2 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 (Velero 1.15) | +| 1.123.1 and later | 1.11.0 to 1.15.1 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 (Velero 1.15) | +| 1.117.3 and later | 1.11.0 to 1.14.1 | 1.27.9, 1.28.9, and 1.29.4 (Velero 1.14) | From e1a355bde334720b2b3247e933ae2fea3f9bf2da Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Wed, 1 Jul 2026 18:53:37 +1200 Subject: [PATCH 03/12] Address Vale passive voice and sentence length suggestions --- docs/enterprise/snapshots-troubleshooting-backup-restore.md | 6 +++--- docs/enterprise/snapshots-velero-cli-installing.md | 4 ++-- docs/enterprise/snapshots-velero-installing-config.mdx | 2 +- docs/enterprise/snapshots-velero-upgrading.mdx | 2 +- docs/partials/snapshots/_node-agent-mem-limit.mdx | 2 +- docs/vendor/snapshots-overview.mdx | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/enterprise/snapshots-troubleshooting-backup-restore.md b/docs/enterprise/snapshots-troubleshooting-backup-restore.md index cf6d462dab..e020991e46 100644 --- a/docs/enterprise/snapshots-troubleshooting-backup-restore.md +++ b/docs/enterprise/snapshots-troubleshooting-backup-restore.md @@ -220,7 +220,7 @@ Add `emptyDir` volumes for `/home/cnb/udmrepo` and `/home/cnb/.cache` to the Vel The Local Volume Provider (LVP) is not compatible with Kopia. If you upgrade to Velero 1.17 or later and the existing storage location uses LVP, snapshots fail because Kopia cannot write to LVP storage. :::important -LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Before you upgrade, migrate to a Kopia-compatible storage destination. For more information, see [Upgrade Velero for Snapshots](snapshots-velero-upgrading). +LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Before you upgrade, migrate to a Kopia-compatible storage destination. For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). ::: #### Solution @@ -228,9 +228,9 @@ LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 Before you upgrade to Velero 1.17 or later, migrate from LVP to a Kopia-compatible destination. Replicated recommends one of the following options: * Reinstall KOTS with `--with-minio=true`. -* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Ensure that the target Velero plugin is installed before you reconfigure the storage location. +* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Install the target Velero plugin before you reconfigure the storage location. -For more information, see [Upgrade Velero for Snapshots](snapshots-velero-upgrading). +For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). ## Snapshot restore is failing diff --git a/docs/enterprise/snapshots-velero-cli-installing.md b/docs/enterprise/snapshots-velero-cli-installing.md index 312aeafd38..06309e017c 100644 --- a/docs/enterprise/snapshots-velero-cli-installing.md +++ b/docs/enterprise/snapshots-velero-cli-installing.md @@ -41,7 +41,7 @@ To install the Velero CLI in an online cluster: curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.18.2/velero-v1.18.2-linux-amd64.tar.gz ``` -1. Run the following command to extract the TAR file: +1. Run the following command to extract the Tape Archive (TAR) file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz @@ -83,7 +83,7 @@ To install the Velero CLI in an air-gapped cluster: 1. Copy the TAR file to the air-gapped node. -1. Run the following command to extract the TAR file: +1. Run the following command to extract the Tape Archive (TAR) file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz diff --git a/docs/enterprise/snapshots-velero-installing-config.mdx b/docs/enterprise/snapshots-velero-installing-config.mdx index e0cee057c3..7ad01742fc 100644 --- a/docs/enterprise/snapshots-velero-installing-config.mdx +++ b/docs/enterprise/snapshots-velero-installing-config.mdx @@ -34,7 +34,7 @@ For more information, see [`velero ensure-permissions`](/reference/kots-cli-vele This section applies to all online and air gap installations. -Velero sets default limits for the velero Pod and the node-agent Pod during installation. There is a known issue with the file-system backup uploader that causes high memory usage, which can result in failures during backup creation when the Pod reaches the memory limit. +Velero sets default limits for the velero Pod and the node-agent Pod during installation. There is a known issue with the file-system backup uploader. High memory usage from this issue can cause failures during backup creation when the Pod reaches the memory limit. diff --git a/docs/enterprise/snapshots-velero-upgrading.mdx b/docs/enterprise/snapshots-velero-upgrading.mdx index 54f3a01733..1cfeb3e54b 100644 --- a/docs/enterprise/snapshots-velero-upgrading.mdx +++ b/docs/enterprise/snapshots-velero-upgrading.mdx @@ -8,7 +8,7 @@ This topic describes how to upgrade Velero for Replicated snapshots. ## Overview -Velero 1.17 and later uses Kopia as the default uploader for file-system backups. If you upgrade from Velero 1.16 or earlier, verify that your storage destination is compatible with Kopia before you upgrade. Some storage destinations, such as the Local Volume Provider (LVP), are not compatible with Kopia and must be migrated before the upgrade. +Velero 1.17 and later uses Kopia as the default uploader for file-system backups. If you upgrade from Velero 1.16 or earlier, verify that your storage destination is compatible with Kopia before you upgrade. Some storage destinations, such as the Local Volume Provider (LVP), are not compatible with Kopia. Migrate these destinations before the upgrade. For more information about snapshot troubleshooting, see [Troubleshoot Snapshots](snapshots-troubleshooting-backup-restore). diff --git a/docs/partials/snapshots/_node-agent-mem-limit.mdx b/docs/partials/snapshots/_node-agent-mem-limit.mdx index 43fa521ef4..e003a1991f 100644 --- a/docs/partials/snapshots/_node-agent-mem-limit.mdx +++ b/docs/partials/snapshots/_node-agent-mem-limit.mdx @@ -14,7 +14,7 @@ kubectl -n velero patch daemonset node-agent -p '{"spec":{"template":{"spec":{"c kubectl -n velero patch daemonset restic -p '{"spec":{"template":{"spec":{"containers":[{"name":"restic","resources":{"limits":{"memory":"2Gi"}}}]}}}}' ``` -Alternatively, you can potentially avoid the node-agent Pod reaching the memory limit during snapshot creation by running the following kubectl command to lower the memory garbage collection target percentage on the node-agent DaemonSet: +Alternatively, you can lower the memory garbage collection target percentage on the node-agent DaemonSet. This can help the node-agent Pod avoid reaching the memory limit during snapshot creation. Run the following kubectl command: **Velero 1.10 and later**: diff --git a/docs/vendor/snapshots-overview.mdx b/docs/vendor/snapshots-overview.mdx index 95f335a40a..3be9f54994 100644 --- a/docs/vendor/snapshots-overview.mdx +++ b/docs/vendor/snapshots-overview.mdx @@ -60,7 +60,7 @@ This section describes the types of backups that are supported with snapshots. F Full backups (also referred to as _instance_ backups) include the KOTS Admin Console and all application data, including application volumes and manifest files. -For clusters created with Replicated kURL, full backups also back up the Docker registry, which is required for air-gapped installations. +For clusters created with Replicated kURL, full backups also back up the Docker registry. Air-gapped installations require this registry. If you manage multiple applications with the Admin Console, data from all applications that support backups is included in a full backup. To be included in full backups, each application must include a manifest file with `kind: Backup` and `apiVersion: velero.io/v1`, which you can check for in the Admin Console. @@ -78,7 +78,7 @@ For disaster recovery, backups should be configured to use a storage destination You can use a storage provider that is compatible with Velero as the storage destination for backups created with the Replicated snapshots feature. For a list of the compatible storage providers, see [Providers](https://velero.io/docs/v1.17/supported-providers/) in the Velero documentation. -You initially configure backups on a supported storage provider backend using the KOTS CLI. If you want to change the storage destination after the initial configuration, you can use the **Snapshots** page in the Admin Console, which has built-in support for the following storage destinations: +You initially configure backups on a supported storage provider backend using the KOTS CLI. To change the storage destination after the initial configuration, use the **Snapshots** page in the Admin Console. The Admin Console supports the following storage destinations: - Amazon Web Services (AWS) - Google Cloud Provider (GCP) @@ -93,7 +93,7 @@ For information about how to configure backup storage destinations for snapshots ### What data is backed up? -Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, this also backs up the Docker registry, which is required for air-gapped installations. +Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, this also backs up the Docker registry. Air-gapped installations require this registry. #### Other object-stored data From 8a611f439d4355cb1372b1c7e38f12f2c5cd0786 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Wed, 1 Jul 2026 19:04:05 +1200 Subject: [PATCH 04/12] Add TAR to Vale TechJargon vocabulary --- docs/enterprise/snapshots-velero-cli-installing.md | 4 ++-- styles/config/vocabularies/TechJargon/accept.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/enterprise/snapshots-velero-cli-installing.md b/docs/enterprise/snapshots-velero-cli-installing.md index 06309e017c..312aeafd38 100644 --- a/docs/enterprise/snapshots-velero-cli-installing.md +++ b/docs/enterprise/snapshots-velero-cli-installing.md @@ -41,7 +41,7 @@ To install the Velero CLI in an online cluster: curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.18.2/velero-v1.18.2-linux-amd64.tar.gz ``` -1. Run the following command to extract the Tape Archive (TAR) file: +1. Run the following command to extract the TAR file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz @@ -83,7 +83,7 @@ To install the Velero CLI in an air-gapped cluster: 1. Copy the TAR file to the air-gapped node. -1. Run the following command to extract the Tape Archive (TAR) file: +1. Run the following command to extract the TAR file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz diff --git a/styles/config/vocabularies/TechJargon/accept.txt b/styles/config/vocabularies/TechJargon/accept.txt index 71eb85d5c1..f9bfdd9783 100644 --- a/styles/config/vocabularies/TechJargon/accept.txt +++ b/styles/config/vocabularies/TechJargon/accept.txt @@ -94,6 +94,7 @@ rollout SAML SCIM SSO +TAR TOTP ttl keyless From 7ae65ef711fd40c5272720a7aa04aa3bb7bb0d3f Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Thu, 2 Jul 2026 11:53:32 +1200 Subject: [PATCH 05/12] Remove Velero version from Kubernetes versions column --- docs/partials/snapshots/_velero-compatibility.mdx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/partials/snapshots/_velero-compatibility.mdx b/docs/partials/snapshots/_velero-compatibility.mdx index 98d7578881..bbaa8602b1 100644 --- a/docs/partials/snapshots/_velero-compatibility.mdx +++ b/docs/partials/snapshots/_velero-compatibility.mdx @@ -5,10 +5,10 @@ The following table lists the versions of Velero that are compatible with each v | KOTS version | Velero version | Supported Kubernetes versions | |------|-------------|-------------| -| TBD and later | 1.11.0 to 1.18.x | 1.33.7, 1.34.1, and 1.35.0 (Velero 1.18) | -| 1.125.2 and later | 1.11.0 to 1.16.2 | 1.31.4, 1.32.3, and 1.33.0 (Velero 1.16) | -| 1.125.0 and later | 1.11.0 to 1.16.1 | 1.31.4, 1.32.3, and 1.33.0 (Velero 1.16) | -| 1.124.18 and later | 1.11.0 to 1.16.0 | 1.31.4, 1.32.3, and 1.33.0 (Velero 1.16) | -| 1.124.0 and later | 1.11.0 to 1.15.2 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 (Velero 1.15) | -| 1.123.1 and later | 1.11.0 to 1.15.1 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 (Velero 1.15) | -| 1.117.3 and later | 1.11.0 to 1.14.1 | 1.27.9, 1.28.9, and 1.29.4 (Velero 1.14) | +| TBD and later | 1.11.0 to 1.18.x | 1.33.7, 1.34.1, and 1.35.0 | +| 1.125.2 and later | 1.11.0 to 1.16.2 | 1.31.4, 1.32.3, and 1.33.0 | +| 1.125.0 and later | 1.11.0 to 1.16.1 | 1.31.4, 1.32.3, and 1.33.0 | +| 1.124.18 and later | 1.11.0 to 1.16.0 | 1.31.4, 1.32.3, and 1.33.0 | +| 1.124.0 and later | 1.11.0 to 1.15.2 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 | +| 1.123.1 and later | 1.11.0 to 1.15.1 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 | +| 1.117.3 and later | 1.11.0 to 1.14.1 | 1.27.9, 1.28.9, and 1.29.4 | From 970a214de8cfe7c303f0d511e9e45cd251f9b205 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Thu, 2 Jul 2026 12:01:18 +1200 Subject: [PATCH 06/12] Remove Velero version references older than 1.10 --- ...napshots-troubleshooting-backup-restore.md | 26 +------------------ docs/vendor/snapshots-overview.mdx | 2 +- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/docs/enterprise/snapshots-troubleshooting-backup-restore.md b/docs/enterprise/snapshots-troubleshooting-backup-restore.md index e020991e46..e838903add 100644 --- a/docs/enterprise/snapshots-troubleshooting-backup-restore.md +++ b/docs/enterprise/snapshots-troubleshooting-backup-restore.md @@ -99,7 +99,7 @@ timed out after 12h0m0s #### Cause -This error message appears when the node-agent Pod operation timeout limit is reached. In Velero v1.4.2 and later, the default timeout is 240 minutes. +This error message appears when the node-agent Pod operation timeout limit is reached. The default timeout is 240 minutes. For Velero 1.16 and earlier, Velero integrates with Restic to provide a solution for backing up and restoring Kubernetes volumes. For more information, see [File System Backup](https://velero.io/docs/v1.10/file-system-backup/) in the Velero documentation. @@ -255,30 +255,6 @@ This issue is fixed in Kubernetes version 1.19. To resolve this issue, upgrade t For more information about the fix, see https://github.com/kubernetes/kubernetes/pull/89937. -### Partial snapshot restore is stuck in progress - -#### Symptom - -In the Admin Console, you see at least one volume restore progress bar frozen at 0%. Example Admin Console display: - -![Snapshot Troubleshoot Frozen Restore](/images/snapshot-troubleshoot-frozen-restore.png) - -You can confirm this is the same issue by running `kubectl get pods -n `, and you should see at least one pod stuck in initialization: - -```shell -NAME READY STATUS RESTARTS AGE -example-mysql-0 0/1 Init:0/2 0 4m15s #<- the offending pod -example-nginx-77b878b4f-zwv2h 3/3 Running 0 4m15s -``` - -#### Cause - -We have seen this issue with Velero version 1.5.4 and opened up this issue with the project to inspect the root cause: https://github.com/vmware-tanzu/velero/issues/3686. However we have not experienced this using Velero 1.6.0 or later. - -#### Solution - -Upgrade Velero to 1.9.0. You can upgrade using Replicated kURL. Or, to follow the Velero upgrade instructions, see [Upgrading to Velero 1.9](https://velero.io/docs/v1.9/upgrade-to-1.9/) in the Velero documentation. - ### Partial snapshot restore finishes with warnings #### Symptom diff --git a/docs/vendor/snapshots-overview.mdx b/docs/vendor/snapshots-overview.mdx index 3be9f54994..60c27a4574 100644 --- a/docs/vendor/snapshots-overview.mdx +++ b/docs/vendor/snapshots-overview.mdx @@ -113,7 +113,7 @@ By default, Velero requires that you opt-in to have pod volumes backed up. In th When you restore an application with snapshots, KOTS first deletes the selected application. All existing application manifests are removed from the cluster, and all `PersistentVolumeClaims` are deleted. This action is not reversible. -Then, the restore process redeploys all of the application manifests. All Pods are given an extra `initContainer` and an extra directory named `.velero`, which are used for restore hooks. For more information about the restore process, see [Restore Reference](https://velero.io/docs/v1.9/restore-reference/) in the Velero documentation. +Then, the restore process redeploys all of the application manifests. All Pods are given an extra `initContainer` and an extra directory named `.velero`, which are used for restore hooks. For more information about the restore process, see [Restore Reference](https://velero.io/docs/v1.18/restore-reference/) in the Velero documentation. When you restore the Admin Console only, no changes are made to the application. From 7d6c3eaff2a999681ce162755821e09f449ffbea Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Thu, 2 Jul 2026 13:14:59 +1200 Subject: [PATCH 07/12] Remove unsupported Velero versions from install partial --- docs/partials/snapshots/_installVelero.mdx | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/docs/partials/snapshots/_installVelero.mdx b/docs/partials/snapshots/_installVelero.mdx index a5fa516433..a8e7a9f246 100644 --- a/docs/partials/snapshots/_installVelero.mdx +++ b/docs/partials/snapshots/_installVelero.mdx @@ -11,7 +11,7 @@ Run one of the following commands to install Velero, depending on the version of --plugins velero/velero-plugin-for-aws:v1.14.2 ``` - * **Velero v1.10 to v1.16**: + * **Velero v1.11 to v1.16**: ```bash velero install \ @@ -21,14 +21,3 @@ Run one of the following commands to install Velero, depending on the version of --use-volume-snapshots=false \ --plugins velero/velero-plugin-for-aws:v1.14.2 ``` - - * **Velero versions earlier than v1.10**: - - ```bash - velero install \ - --no-default-backup-location \ - --no-secret \ - --use-restic \ - --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.14.2 - ``` From 07f4caf45dfe1e8766a2104e779d8df40a60f286 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Thu, 2 Jul 2026 14:37:23 +1200 Subject: [PATCH 08/12] Address Vale review comments from PR #4221 --- ...napshots-troubleshooting-backup-restore.md | 32 ++++++++--------- .../snapshots-velero-cli-installing.md | 6 ++-- .../snapshots-velero-installing-config.mdx | 4 +-- .../enterprise/snapshots-velero-upgrading.mdx | 10 +++--- .../snapshots/_node-agent-mem-limit.mdx | 18 +--------- docs/vendor/snapshots-overview.mdx | 34 +++++++++---------- styles/Replicated/Acronyms.yml | 1 + 7 files changed, 45 insertions(+), 60 deletions(-) diff --git a/docs/enterprise/snapshots-troubleshooting-backup-restore.md b/docs/enterprise/snapshots-troubleshooting-backup-restore.md index e838903add..75ca6b60f9 100644 --- a/docs/enterprise/snapshots-troubleshooting-backup-restore.md +++ b/docs/enterprise/snapshots-troubleshooting-backup-restore.md @@ -3,7 +3,7 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" # Troubleshoot snapshots -When a snapshot fails, a support bundle will be collected and stored automatically. Because this is a point-in-time collection of all logs and system state at the time of the failed snapshot, this is a good place to view the logs. +When a snapshot fails, KOTS automatically collects and stores a support bundle. This bundle contains all logs and system state at the time of the failure. It is a good place to view the logs. @@ -29,7 +29,7 @@ An error occurred: some backup storage locations are invalid: backup store for l #### Cause -If the cloud access credentials are invalid or do not have access to the location in the configuration, Velero will crashloop. The Velero logs will be included in a support bundle, and the message will look like this. +If the cloud access credentials are invalid or do not have access to the location in the configuration, Velero will crashloop. The support bundle includes the Velero logs, and the message looks like this. #### Solution @@ -53,7 +53,7 @@ An error occurred: some backup storage locations are invalid: backup store for l #### Cause -This error message is caused when Velero is attempting to start, and it is configured to use a reconfigured or re-used bucket. +Velero displays this error message when it attempts to start and uses a reconfigured or re-used bucket. When configuring Velero to use a bucket, the bucket cannot contain other data, or Velero will crash. @@ -78,7 +78,7 @@ time="2023-11-16T21:29:44Z" level=fatal msg="Failed to start metric server for n #### Cause -This is a result of a known issue in Velero 1.12.0 and 1.12.1 where the port is not set correctly when starting the metrics server. This causes the metrics server to fail to start with a `permission denied` error in environments that do not run MinIO and have Host Path, NFS, or internal storage destinations configured. When the metrics server fails to start, the node-agent Pod crashes. For more information about this issue, see [the GitHub issue details](https://github.com/vmware-tanzu/velero/issues/6792). +This issue occurs in Velero 1.12.0 and 1.12.1. Velero does not set the port correctly when starting the metrics server. The metrics server fails to start with a `permission denied` error. The error occurs in environments that do not run MinIO and have Host Path, Network File System (NFS), or internal storage destinations configured. When the metrics server fails to start, the node-agent Pod crashes. For more information about this issue, see [the GitHub issue details](https://github.com/vmware-tanzu/velero/issues/6792). #### Solution @@ -99,7 +99,7 @@ timed out after 12h0m0s #### Cause -This error message appears when the node-agent Pod operation timeout limit is reached. The default timeout is 240 minutes. +This error message appears when the node-agent Pod operation reaches the timeout limit. The default timeout is 240 minutes. For Velero 1.16 and earlier, Velero integrates with Restic to provide a solution for backing up and restoring Kubernetes volumes. For more information, see [File System Backup](https://velero.io/docs/v1.10/file-system-backup/) in the Velero documentation. @@ -131,7 +131,7 @@ The timeout value reverts back to the default value if you rerun the `velero ins #### Symptom -The node-agent Pod is killed by the Linux kernel Out Of Memory (OOM) killer or snapshots are failing with errors similar to: +The Linux kernel Out Of Memory (OOM) killer kills the node-agent Pod, or snapshots fail with errors similar to: ``` pod volume backup failed: ... signal: killed @@ -139,7 +139,7 @@ pod volume backup failed: ... signal: killed #### Cause -Velero sets default limits for the velero Pod and the node-agent Pod during installation. For Velero 1.16 and earlier, there is a known issue with Restic that causes high memory usage, which can result in failures during snapshot creation when the Pod reaches the memory limit. For Velero 1.17 and later, Velero uses Kopia for file-system backups by default, and large volumes can also require a higher memory limit. +Velero sets default limits for the velero Pod and the node-agent Pod during installation. For Velero 1.16 and earlier, a known Restic issue causes high memory usage. This high memory usage can cause failures during snapshot creation when the Pod reaches the memory limit. For Velero 1.17 and later, Velero uses Kopia for file-system backups by default. Large volumes with Kopia can also require a higher memory limit. For more information about the Restic issue, see the [Restic backup — OOM-killed on raspberry pi after backing up another computer to same repo](https://github.com/restic/restic/issues/1988) issue in the Restic GitHub repository. @@ -147,7 +147,7 @@ For more information about the Restic issue, see the [Restic backup — OOM-kill -### At least one source file could not be read +### Velero cannot read at least one source file #### Symptom @@ -179,7 +179,7 @@ For Velero 1.17 and later, Kopia spawns data mover pods from the node-agent. If #### Solution -Check the node-agent logs for errors that prevent data mover pods from starting. Verify that the data mover pod image can be pulled and that any pod security policies or security context constraints allow the pod to start. +Check the node-agent logs for errors that prevent data mover pods from starting. Verify that the cluster can pull the data mover pod image and that any pod security policies or security context constraints allow the pod to start. ### BackupRepository is not available @@ -207,7 +207,7 @@ Common errors include invalid credentials, network connectivity issues, or probl #### Cause -For Velero 1.17 and later, Kopia needs writable directories for cache and configuration. The default paths are `/home/cnb/udmrepo` and `/home/cnb/.cache`. If `ReadOnlyRootFilesystem` is set for the Velero or node-agent pods, Kopia cannot write to these directories and the backup or restore fails. +For Velero 1.17 and later, Kopia needs writable directories for cache and configuration. The default paths are `/home/cnb/udmrepo` and `/home/cnb/.cache`. If `ReadOnlyRootFilesystem` applies to the Velero or node-agent pods, Kopia cannot write to these directories and the backup or restore fails. #### Solution @@ -239,7 +239,7 @@ For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrad #### Symptom -In the Replicated KOTS Admin Console, you see an **Application failed to restore** error message that indicates the port number for a static NodePort is already in use. For example: +In the Replicated KOTS Admin Console, you see an **Application failed to restore** error. The error indicates that the port number for a static NodePort is already in use. For example: ![Snapshot Troubleshoot Service NodePort](/images/snapshot-troubleshoot-service-nodeport.png) @@ -247,11 +247,11 @@ In the Replicated KOTS Admin Console, you see an **Application failed to restore #### Cause -There is a known issue in Kubernetes versions earlier than version 1.19 where using a static NodePort for services can collide in multi-primary high availability setups when recreating the services. For more information about this known issue, see https://github.com/kubernetes/kubernetes/issues/85894. +A known issue in Kubernetes versions earlier than version 1.19 can cause static NodePort services to collide in multi-primary high availability setups. This collision occurs when recreating the services. For more information about this known issue, see https://github.com/kubernetes/kubernetes/issues/85894. #### Solution -This issue is fixed in Kubernetes version 1.19. To resolve this issue, upgrade to Kubernetes version 1.19 or later. +Kubernetes version 1.19 fixes this issue. To resolve this issue, upgrade to Kubernetes version 1.19 or later. For more information about the fix, see https://github.com/kubernetes/kubernetes/pull/89937. @@ -259,14 +259,14 @@ For more information about the fix, see https://github.com/kubernetes/kubernetes #### Symptom -In the Admin Console, when the partial snapshot restore completes, you see warnings indicating that Endpoint resources were not restored: +In the Admin Console, when the partial snapshot restore completes, you see warnings indicating that Velero did not restore Endpoint resources: ![Snapshot Troubleshoot Restore Warnings](/images/snapshot-troubleshoot-restore-warnings.png) #### Cause -The resource restore priority was changed in Velero 1.10.3 and 1.11.0, which leads to this warning when restoring Endpoint resources. For more information about this issue, see [the issue details](https://github.com/vmware-tanzu/velero/issues/6280) in GitHub. +Velero changed the resource restore priority in 1.10.3 and 1.11.0, which leads to this warning when restoring Endpoint resources. For more information about this issue, see [the issue details](https://github.com/vmware-tanzu/velero/issues/6280) in GitHub. #### Solution -These warnings do not necessarily mean that the restore itself failed. The endpoints likely do exist as they are created by Kubernetes when the related Service resources were restored. However, to prevent encountering these warnings, use Velero version 1.11.1 or later. +These warnings do not necessarily mean that the restore itself failed. The endpoints likely exist because Kubernetes creates them when the restore process restores the related Service resources. However, to prevent encountering these warnings, use Velero version 1.11.1 or later. diff --git a/docs/enterprise/snapshots-velero-cli-installing.md b/docs/enterprise/snapshots-velero-cli-installing.md index 312aeafd38..96e5cc6f6e 100644 --- a/docs/enterprise/snapshots-velero-cli-installing.md +++ b/docs/enterprise/snapshots-velero-cli-installing.md @@ -5,7 +5,7 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" You install the Velero CLI before installing Velero and configuring a storage destination for backups. :::note -For embedded clusters created with Replicated kURL, if the kURL Installer spec included the Velero add-on, then Velero was automatically installed with default internal storage. Replicated recommends that you proceed to change the default internal storage because it is insufficient for disaster recovery. See [Updating Storage Settings in the Admin Console](snapshots-updating-with-admin-console). +For embedded clusters created with Replicated kURL, check whether the kURL Installer spec included the Velero add-on. If so, the kURL installer automatically installed Velero with default internal storage. Replicated recommends that you change the default internal storage because it is insufficient for disaster recovery. See [Updating Storage Settings in the Admin Console](snapshots-updating-with-admin-console). ::: @@ -23,7 +23,7 @@ To install the Velero CLI in an online cluster: - (Embedded kURL cluster) Run an SSH command to access and authenticate to your cluster node. - (Existing cluster) Open a terminal in the environment that you manage the cluster from, which can be a local machine that has kubectl installed. -1. Check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although earlier versions of Velero are supported, Replicated recommends using the latest supported version. For more information about supported versions, see [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). +1. Check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although Replicated supports earlier versions of Velero, Replicated recommends using the latest supported version. For more information about supported versions, see [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). Note the version number for the next step. @@ -63,7 +63,7 @@ To install the Velero CLI in an online cluster: To install the Velero CLI in an air-gapped cluster: -1. From a computer with internet access, check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although earlier versions of Velero are supported, Replicated recommends using the latest supported version. See [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). +1. From a computer with internet access, check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although Replicated supports earlier versions of Velero, Replicated recommends using the latest supported version. See [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). Note the version number for the next step. diff --git a/docs/enterprise/snapshots-velero-installing-config.mdx b/docs/enterprise/snapshots-velero-installing-config.mdx index 7ad01742fc..168c123bf9 100644 --- a/docs/enterprise/snapshots-velero-installing-config.mdx +++ b/docs/enterprise/snapshots-velero-installing-config.mdx @@ -11,7 +11,7 @@ This topic describes how to configure namespace access and the memory limit for ## Overview -The Replicated KOTS Admin Console requires access to the namespace where Velero is installed. If your Admin Console is running with minimal role-based-access-control (RBAC) privileges, you must enable the Admin Console to access Velero. +The Replicated KOTS Admin Console requires access to the namespace where Velero runs. If your Admin Console is running with minimal role-based-access-control (RBAC) privileges, you must enable the Admin Console to access Velero. Additionally, if the application uses a large amount of memory, you can configure the default memory limit to help ensure that Velero runs successfully with snapshots. @@ -26,7 +26,7 @@ kubectl kots velero ensure-permissions --namespace ADMIN_CONSOLE_NAMESPACE --vel ``` Replace: * `ADMIN_CONSOLE_NAMESPACE` with the namespace on the cluster where the Admin Console is running. -* `VELERO_NAMESPACE` with the namespace on the cluster where Velero is installed. +* `VELERO_NAMESPACE` with the namespace on the cluster where Velero runs. For more information, see [`velero ensure-permissions`](/reference/kots-cli-velero-ensure-permissions/) in the KOTS CLI documentation. For more information about RBAC privileges for the Admin Console, see [Kubernetes RBAC](/vendor/packaging-rbac). diff --git a/docs/enterprise/snapshots-velero-upgrading.mdx b/docs/enterprise/snapshots-velero-upgrading.mdx index 1cfeb3e54b..e1bcd55e9a 100644 --- a/docs/enterprise/snapshots-velero-upgrading.mdx +++ b/docs/enterprise/snapshots-velero-upgrading.mdx @@ -20,11 +20,11 @@ Complete the following prerequisites before you upgrade Velero: 1. Take a snapshot of the application before you upgrade. This snapshot lets you restore the application if the upgrade fails. -1. Ensure that the Velero plugin for the target storage destination is installed. For example, verify that the plugin image is available in the registry for air-gapped environments. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). +1. Install the Velero plugin for the target storage destination, if needed. For example, verify that the plugin image is available in the registry for air-gapped environments. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). ## Migrate from local volume provider (LVP) to a Kopia-compatible destination -LVP is not compatible with Kopia. If Velero is configured to use LVP, you must migrate to a Kopia-compatible destination before you upgrade to Velero 1.17 or later. +LVP is not compatible with Kopia. If the storage location uses LVP, migrate to a Kopia-compatible destination before you upgrade to Velero 1.17 or later. :::important LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Replicated recommends that you migrate to a Kopia-compatible destination before you upgrade. @@ -35,7 +35,7 @@ In-place reconfiguration from LVP to a filesystem-based Minio deployment is not Replicated recommends one of the following options: * Reinstall KOTS with `--with-minio=true`. This installs a Minio object store that is compatible with Kopia. -* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Ensure that the target Velero plugin is installed before you reconfigure the storage location. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). +* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Install the target Velero plugin before you reconfigure the storage location. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). For more information about LVP storage issues after upgrade, see [LVP storage location is unavailable after upgrade](snapshots-troubleshooting-backup-restore#lvp-storage-location-is-unavailable-after-upgrade) in _Troubleshoot Snapshots_. @@ -49,7 +49,7 @@ To upgrade Velero, follow the upgrade instructions for the target Velero version After you upgrade Velero, verify that the upgrade is successful: -1. Run `velero version` to confirm that the expected Velero version is installed. +1. Run `velero version` to confirm the expected Velero version. 1. Check the status of the `BackupStorageLocation`: @@ -79,7 +79,7 @@ After you upgrade Velero, verify that the upgrade is successful: ## Restore existing Restic backups -Restic backups created on Velero 1.16 and earlier can be restored on Velero 1.17 and 1.18. Velero 1.17 and 1.18 retain the ability to restore Restic backups. To restore an existing Restic backup, use the same storage location and credentials that were used to create the backup. +Velero 1.17 and 1.18 can restore Restic backups created on Velero 1.16 and earlier. To restore an existing Restic backup, use the same storage location and credentials that you used to create the backup. ## Additional resources diff --git a/docs/partials/snapshots/_node-agent-mem-limit.mdx b/docs/partials/snapshots/_node-agent-mem-limit.mdx index e003a1991f..0c25dc384e 100644 --- a/docs/partials/snapshots/_node-agent-mem-limit.mdx +++ b/docs/partials/snapshots/_node-agent-mem-limit.mdx @@ -1,29 +1,13 @@ Increase the default memory limit for the node-agent Pod if your application is particularly large. For more information about configuring Velero resource requests and limits, see [Customize resource requests and limits](https://velero.io/docs/v1.17/customize-installation/#customize-resource-requests-and-limits) in the Velero documentation. -For example, the following kubectl commands will increase the memory limit for the node-agent DaemonSet from the default of 1Gi to 2Gi. - -**Velero 1.10 and later**: +For example, the following kubectl command increases the memory limit for the node-agent DaemonSet from the default of 1Gi to 2Gi: ``` kubectl -n velero patch daemonset node-agent -p '{"spec":{"template":{"spec":{"containers":[{"name":"node-agent","resources":{"limits":{"memory":"2Gi"}}}]}}}}' ``` -**Velero versions earlier than 1.10**: - -``` -kubectl -n velero patch daemonset restic -p '{"spec":{"template":{"spec":{"containers":[{"name":"restic","resources":{"limits":{"memory":"2Gi"}}}]}}}}' -``` - Alternatively, you can lower the memory garbage collection target percentage on the node-agent DaemonSet. This can help the node-agent Pod avoid reaching the memory limit during snapshot creation. Run the following kubectl command: -**Velero 1.10 and later**: - ``` kubectl -n velero set env daemonset/node-agent GOGC=1 -``` - -**Velero versions earlier than 1.10**: - -``` -kubectl -n velero set env daemonset/restic GOGC=1 ``` \ No newline at end of file diff --git a/docs/vendor/snapshots-overview.mdx b/docs/vendor/snapshots-overview.mdx index 60c27a4574..90efd22ba8 100644 --- a/docs/vendor/snapshots-overview.mdx +++ b/docs/vendor/snapshots-overview.mdx @@ -10,7 +10,7 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" # About backup and restore with snapshots -This topic provides an introduction to the Replicated KOTS snapshots feature for backup and restore. It describes how vendors enable snapshots, the type of data that is backed up, and how to troubleshoot issues for enterprise users. +This topic provides an introduction to the Replicated KOTS snapshots feature for backup and restore. It describes how vendors enable snapshots, the types of data that snapshots back up, and how to troubleshoot issues for enterprise users. :::note @@ -22,11 +22,11 @@ This topic provides an introduction to the Replicated KOTS snapshots feature for An important part of the lifecycle of an application is backup and restore. You can enable Replicated KOTS snapshots to support backup and restore for existing cluster installations with KOTS and Replicated kURL installations. -When snapshots is enabled for your application, your customers can manage and perform backup and restore from the Admin Console or KOTS CLI. +When you enable snapshots for your application, your customers can manage and perform backup and restore from the Admin Console or KOTS CLI. Snapshots uses the Velero open source project as the backend to back up Kubernetes manifests and persistent volumes. Velero is a mature, fully-featured application. For more information, see the [Velero documentation](https://velero.io/docs/). -In addition to the default functionality that Velero provides, KOTS exposes hooks that let you inject scripts that can execute both before and after a backup, and before and after a restore. For more information, see [Configure Backup and Restore Hooks for Snapshots](/vendor/snapshots-hooks). +In addition to the default functionality that Velero provides, KOTS exposes hooks that let you inject scripts. These scripts can execute both before and after a backup, and before and after a restore. For more information, see [Configure Backup and Restore Hooks for Snapshots](/vendor/snapshots-hooks). ### Limitations and considerations @@ -34,7 +34,7 @@ In addition to the default functionality that Velero provides, KOTS exposes hook - The snapshots feature is available only for licenses with the **Allow Snapshots** option enabled. For more information, see [Create and Manage Customers](/vendor/releases-creating-customer). -- Snapshots are useful for rollback and disaster recovery scenarios. They are not intended to be used for application migration. +- Snapshots are useful for rollback and disaster recovery scenarios. Do not use them for application migration. - @@ -54,7 +54,7 @@ In addition to the default functionality that Velero provides, KOTS exposes hook ## About backups -This section describes the types of backups that are supported with snapshots. For information about how to configure backup storage destinations for snapshots, see the [Configuring Backup Storage](/enterprise/snapshots-velero-cli-installing) section. +This section describes the backup types that the snapshots feature supports. For information about how to configure backup storage destinations for snapshots, see the [Configuring Backup Storage](/enterprise/snapshots-velero-cli-installing) section. ### Application and Admin Console (full) backups @@ -62,19 +62,19 @@ Full backups (also referred to as _instance_ backups) include the KOTS Admin Con For clusters created with Replicated kURL, full backups also back up the Docker registry. Air-gapped installations require this registry. -If you manage multiple applications with the Admin Console, data from all applications that support backups is included in a full backup. To be included in full backups, each application must include a manifest file with `kind: Backup` and `apiVersion: velero.io/v1`, which you can check for in the Admin Console. +If you manage multiple applications with the Admin Console, a full backup includes data from all applications that support backups. Each application must include a manifest file with `kind: Backup` and `apiVersion: velero.io/v1`. KOTS includes the application in full backups only when this manifest file is present. You can check for this manifest file in the Admin Console. -Full backups are recommended because they support all types of restores. For example, you can restore both the Admin Console and application from a full backup to a new cluster in disaster recovery scenarios. Or, you can use a full backup to restore only application data for the purpose of rolling back after deploying a new version of an application. +Replicated recommends full backups because they support all types of restores. For example, you can restore both the Admin Console and application from a full backup to a new cluster in disaster recovery scenarios. Or, you can use a full backup to restore only application data for the purpose of rolling back after deploying a new version of an application. ### Application-only (partial) backups Partial backups back up the application volumes and manifest files only. Partial backups do not back up the KOTS Admin Console. -Partial backups can be useful if you need to roll back after deploying a new application version. Partial backups of the application only _cannot_ be restored to a new cluster, and are therefore not useable for disaster recovery scenarios. +Partial backups can be useful if you need to roll back after deploying a new application version. You cannot restore partial backups of the application only to a new cluster, so they are not usable for disaster recovery scenarios. ### Backup storage destinations -For disaster recovery, backups should be configured to use a storage destination that exists outside of the cluster. This is especially true for installations in clusters created with Replicated kURL, because the default storage location on these clusters is internal. +For disaster recovery, configure backups to use a storage destination that exists outside of the cluster. This is especially true for installations in clusters created with Replicated kURL, because the default storage location on these clusters is internal. You can use a storage provider that is compatible with Velero as the storage destination for backups created with the Replicated snapshots feature. For a list of the compatible storage providers, see [Providers](https://velero.io/docs/v1.17/supported-providers/) in the Velero documentation. @@ -91,15 +91,15 @@ kURL installers that include the Velero add-on also include a locally-provisione For information about how to configure backup storage destinations for snapshots, see the [Configuring Backup Storage](/enterprise/snapshots-velero-cli-installing) section. -### What data is backed up? +### What data do backups include? -Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, this also backs up the Docker registry. Air-gapped installations require this registry. +Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, full backups also back up the Docker registry. Air-gapped installations require this registry. #### Other object-stored data For kURL clusters, you might be using object-stored data that is not specific to the kURL KOTS add-on. -For object-stored data that is not KOTS-specific and does not use persistentVolumeClaims (PVCs), you must write custom backup and restore hooks to enable back ups for that object-stored data. For example, Rook and Ceph do not use PVCs and so require custom backup and restore hooks. For more information about writing custom hooks, see [Configure Backup and Restore Hooks for Snapshots](snapshots-hooks). +For object-stored data that is not KOTS-specific and does not use persistentVolumeClaims (PVCs), you must write custom backup and restore hooks. For example, Rook, Ceph, and similar storage providers do not use PVCs and require custom backup and restore hooks. For more information about writing custom hooks, see [Configure Backup and Restore Hooks for Snapshots](snapshots-hooks). #### Pod volume data @@ -111,11 +111,11 @@ By default, Velero requires that you opt-in to have pod volumes backed up. In th -When you restore an application with snapshots, KOTS first deletes the selected application. All existing application manifests are removed from the cluster, and all `PersistentVolumeClaims` are deleted. This action is not reversible. +When you restore an application with snapshots, KOTS first deletes the selected application. KOTS removes all existing application manifests from the cluster and deletes all `PersistentVolumeClaims`. This action is not reversible. -Then, the restore process redeploys all of the application manifests. All Pods are given an extra `initContainer` and an extra directory named `.velero`, which are used for restore hooks. For more information about the restore process, see [Restore Reference](https://velero.io/docs/v1.18/restore-reference/) in the Velero documentation. +Then, the restore process redeploys all of the application manifests. The restore process gives all Pods an extra `initContainer` and an extra directory named `.velero`. KOTS uses these for restore hooks. For more information about the restore process, see [Restore Reference](https://velero.io/docs/v1.18/restore-reference/) in the Velero documentation. -When you restore the Admin Console only, no changes are made to the application. +When you restore the Admin Console only, the restore process makes no changes to the application. For information about how to restore using the Admin Console or the KOTS CLI, see [Restore from Backups](/enterprise/snapshots-restoring-full). @@ -127,7 +127,7 @@ This section provides an overview of how vendors and enterprise users can config To enable the snapshots backup and restore feature for your users, you must: -- Have the snapshots entitlement enabled in your Replicated vendor account. For account entitlements, contact the Replicated TAM team. +- Have the snapshots entitlement enabled in your Replicated vendor account. For account entitlements, contact the Replicated Technical Account Manager (TAM) team. - Define a manifest for creating backups. See [Configure Snapshots](snapshots-configuring-backups). - When needed, configure backup and restore hooks. See [Configure Backup and Restore Hooks for Snapshots](snapshots-hooks). - Enable the **Allow Snapshot** option in customer licenses. See [Create and Manage Customers](releases-creating-customer). @@ -142,7 +142,7 @@ From a full backup, users restore using the KOTS CLI or the Admin Console as ind -Partial backups are not recommended as they are a legacy feature and only back up the application volumes and manifests. Partial backups can be restored only from the Admin Console. +Partial backups are not recommended as they are a legacy feature and only back up the application volumes and manifests. Users can restore partial backups only from the Admin Console. ### Troubleshooting snapshots diff --git a/styles/Replicated/Acronyms.yml b/styles/Replicated/Acronyms.yml index a9c1db766b..4ab1179f70 100644 --- a/styles/Replicated/Acronyms.yml +++ b/styles/Replicated/Acronyms.yml @@ -57,6 +57,7 @@ exceptions: - SSH - SSL - SVG + - TAR - TBD - TCP - TODO From 21a2a5627fcae3d4ace140ddb46f6f5108b5bf66 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Thu, 2 Jul 2026 15:32:11 +1200 Subject: [PATCH 09/12] Add troubleshooting topic for reclaiming snapshot disk space --- ...napshots-troubleshooting-reclaim-space.mdx | 81 +++++++++++++++++++ sidebars.js | 1 + 2 files changed, 82 insertions(+) create mode 100644 docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx diff --git a/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx b/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx new file mode 100644 index 0000000000..818cec40d0 --- /dev/null +++ b/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx @@ -0,0 +1,81 @@ +import KotsAvailability from "../partials/kots/_kots-availability.mdx" + +# Deleted snapshots do not reclaim disk space in embedded kURL clusters + +This topic describes how to reclaim disk space after deleting snapshots in embedded kURL clusters when the expected space is not freed. + + + +## Symptom + +After deleting snapshots from the Admin Console or with the KOTS CLI, the disk space on the embedded kURL cluster is not reclaimed. The snapshots continue to consume disk space. The available disk space might remain the same or decrease over time. + +## Cause + +The cause depends on the version of Velero installed in the cluster. + +### Velero 1.16 and earlier + +Velero 1.16 and earlier use Restic for file-system backups. Restic does not immediately remove data from the backup repository when you delete a snapshot. Instead, Restic marks the data as no longer referenced and reclaims the space only during a prune operation. By default, Velero runs a prune operation based on the `--default-restic-prune-frequency` flag. If the prune frequency is too low or pruning has not run, deleted snapshots continue to consume disk space. + +In addition, Velero versions up to and including 1.16.x do not purge data from Host Path storage. Even if the Restic prune operation runs, the Host Path storage location might not release the underlying disk space. + +### Velero 1.17 and later + +Velero 1.17 and later use Kopia for file-system backups by default. Velero replaced the `--default-restic-prune-frequency` flag with `--default-repo-maintain-frequency`. Kopia performs repository maintenance that reclaims space from deleted snapshots. Velero 1.17 and later also include the fix for purging data from Host Path storage. + +## Solution + +The solution depends on the Velero version. + +### Velero 1.16 and earlier + +For Velero 1.16 and earlier, you can manually reclaim disk space by running a Restic prune. You can also adjust the prune frequency so that Velero reclaims space automatically. + +To run a manual prune: + +1. Run the following command to create a `DeleteBackupRequests` or use the Velero CLI to trigger a Restic prune. The exact command depends on your Velero version and configuration. For more information, see [Restic backup repository maintenance](https://velero.io/docs/v1.16/file-system-backup/) in the Velero documentation. + +1. Monitor the node-agent Pod logs to verify that the prune operation completes: + + ```bash + kubectl logs -n velero -l name=node-agent -f + ``` + +To adjust the prune frequency, patch the Velero deployment with the `--default-restic-prune-frequency` flag. For example: + +```bash +kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--default-restic-prune-frequency=24h0m0s"}]' +``` + +Replace `24h0m0s` with the desired frequency. Velero uses the default value if you do not set the flag. For more information, see [File System Backup](https://velero.io/docs/v1.16/file-system-backup/) in the Velero documentation. + +### Velero 1.17 and later + +For Velero 1.17 and later, Kopia automatically performs repository maintenance based on the `--default-repo-maintain-frequency` flag. If disk space is not reclaimed, verify that the Kopia maintenance job has run. + +To check the maintenance status, run: + +```bash +velero repo get +``` + +Describe the `BackupRepository` CRs if they are not in a `Ready` state: + +```bash +kubectl describe backuprepository BACKUP_REPOSITORY_NAME -n velero +``` + +To adjust the maintenance frequency, patch the Velero deployment with the `--default-repo-maintain-frequency` flag. For example: + +```bash +kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--default-repo-maintain-frequency=24h0m0s"}]' +``` + +For more information, see [File System Backup](https://velero.io/docs/v1.17/file-system-backup/) in the Velero documentation. + +## Additional resources + +- [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility) +- [Troubleshoot Snapshots](/enterprise/snapshots-troubleshooting-backup-restore) +- [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading) diff --git a/sidebars.js b/sidebars.js index d6fef8284c..4261529f7f 100644 --- a/sidebars.js +++ b/sidebars.js @@ -408,6 +408,7 @@ const sidebars = { "enterprise/snapshots-restoring-full", "enterprise/snapshots-updating-with-admin-console", "enterprise/snapshots-troubleshooting-backup-restore", + "enterprise/snapshots-troubleshooting-reclaim-space", "enterprise/snapshots-velero-upgrading", ], }, From eb0d1b74ccd97e1d036e68f214b3696f1ed54c27 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Thu, 2 Jul 2026 16:23:12 +1200 Subject: [PATCH 10/12] Refine snapshot disk space topic with securityContext and flag details --- ...napshots-troubleshooting-reclaim-space.mdx | 73 +++++++++---------- styles/Replicated/Acronyms.yml | 1 + 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx b/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx index 818cec40d0..e0f037a0fe 100644 --- a/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx +++ b/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx @@ -1,81 +1,78 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" -# Deleted snapshots do not reclaim disk space in embedded kURL clusters +# Deleted snapshots do not reclaim disk space on host path storage -This topic describes how to reclaim disk space after deleting snapshots in embedded kURL clusters when the expected space is not freed. +This topic describes why deleted snapshots might not reclaim disk space on embedded kURL clusters. It also explains how to reclaim the space manually. + +The issue applies to clusters that use Host Path storage with the Velero Restic uploader. ## Symptom -After deleting snapshots from the Admin Console or with the KOTS CLI, the disk space on the embedded kURL cluster is not reclaimed. The snapshots continue to consume disk space. The available disk space might remain the same or decrease over time. +On an embedded kURL cluster that uses Host Path storage with the Velero Restic uploader, disk space is not reclaimed after you delete snapshots. You can delete snapshots from the Admin Console or with the KOTS CLI. The disk continues to fill even though only the retained backups appear in `velero get backup`. ## Cause -The cause depends on the version of Velero installed in the cluster. +Velero's repository maintenance job runs a `restic prune` operation. The prune removes data that is no longer referenced from the Restic repository. This frees the disk space that deleted or expired snapshots used. -### Velero 1.16 and earlier +On Velero versions earlier than 1.17.0, the repository maintenance job does not inherit the Velero Pod security context. The job runs as a different user than the node-agent Pod that created the Restic repository files on the Host Path store. The Restic files belong to UID 1001 with mode 0700. The maintenance job receives a `permission denied` error when it tries to read the repository. The prune operation then fails. -Velero 1.16 and earlier use Restic for file-system backups. Restic does not immediately remove data from the backup repository when you delete a snapshot. Instead, Restic marks the data as no longer referenced and reclaims the space only during a prune operation. By default, Velero runs a prune operation based on the `--default-restic-prune-frequency` flag. If the prune frequency is too low or pruning has not run, deleted snapshots continue to consume disk space. +Because the prune fails, data from deleted or expired snapshots is never reclaimed. The backup disk grows without bound. -In addition, Velero versions up to and including 1.16.x do not purge data from Host Path storage. Even if the Restic prune operation runs, the Host Path storage location might not release the underlying disk space. +For example, the maintenance Pod logs show an error similar to the following: -### Velero 1.17 and later +``` +restic prune --repo=.../restic/default ... +Fatal: unable to open repository at .../restic/default: + ReadDir: open .../restic/default/keys: permission denied +``` -Velero 1.17 and later use Kopia for file-system backups by default. Velero replaced the `--default-restic-prune-frequency` flag with `--default-repo-maintain-frequency`. Kopia performs repository maintenance that reclaims space from deleted snapshots. Velero 1.17 and later also include the fix for purging data from Host Path storage. +Velero 1.17.0 and later include the upstream fix. The fix copies the security context from the origin Pod to the maintenance job. The automated prune operation can then read the repository files and reclaim disk space. For more information, see [Copy security context from origin pod](https://github.com/vmware-tanzu/velero/pull/8943) in the Velero repository. ## Solution The solution depends on the Velero version. -### Velero 1.16 and earlier - -For Velero 1.16 and earlier, you can manually reclaim disk space by running a Restic prune. You can also adjust the prune frequency so that Velero reclaims space automatically. - -To run a manual prune: - -1. Run the following command to create a `DeleteBackupRequests` or use the Velero CLI to trigger a Restic prune. The exact command depends on your Velero version and configuration. For more information, see [Restic backup repository maintenance](https://velero.io/docs/v1.16/file-system-backup/) in the Velero documentation. +### Velero 1.17.0 and later -1. Monitor the node-agent Pod logs to verify that the prune operation completes: - - ```bash - kubectl logs -n velero -l name=node-agent -f - ``` - -To adjust the prune frequency, patch the Velero deployment with the `--default-restic-prune-frequency` flag. For example: +Upgrade to Velero 1.17.0 or later. The automated Restic repository maintenance job can then read the repository files and reclaim disk space. After the upgrade, you can adjust the maintenance frequency with the `--default-repo-maintain-frequency` Velero server flag. For example: ```bash -kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--default-restic-prune-frequency=24h0m0s"}]' +kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--default-repo-maintain-frequency=48h0m0s"}]' ``` -Replace `24h0m0s` with the desired frequency. Velero uses the default value if you do not set the flag. For more information, see [File System Backup](https://velero.io/docs/v1.16/file-system-backup/) in the Velero documentation. +Replace `48h0m0s` with the desired frequency. Velero uses the default value if you do not set the flag. -### Velero 1.17 and later +:::note +The `--default-restic-prune-frequency` flag was the previous name for this flag. Velero removed `--default-restic-prune-frequency` in version 1.10.0. It renamed the flag to `--default-repo-maintain-frequency`. On Velero 1.10.0 and later, `--default-restic-prune-frequency` is not a recognized flag and has no effect. For more information, see the [Velero 1.10 breaking changes](https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.10.md#breaking-changes). +::: -For Velero 1.17 and later, Kopia automatically performs repository maintenance based on the `--default-repo-maintain-frequency` flag. If disk space is not reclaimed, verify that the Kopia maintenance job has run. +For more information about file-system backups, see [File System Backup](https://velero.io/docs/v1.17/file-system-backup/) in the Velero documentation. -To check the maintenance status, run: +### Velero 1.16.x and earlier -```bash -velero repo get -``` +On Velero versions earlier than 1.17.0, the automated prune fails regardless of the maintenance frequency. Tuning the frequency does not help. To reclaim disk space until you can upgrade to Velero 1.17.0 or later, run `restic prune` manually from inside the Velero Pod. The Velero Pod runs as the correct user (UID 1001), so the prune operation can read the repository files. -Describe the `BackupRepository` CRs if they are not in a `Ready` state: +Run the following commands to prune both the `default` and `kurl` Restic repositories: ```bash -kubectl describe backuprepository BACKUP_REPOSITORY_NAME -n velero -``` +PREFIX=$(kubectl -n velero get bsl default -o jsonpath='{.spec.config.resticRepoPrefix}') -To adjust the maintenance frequency, patch the Velero deployment with the `--default-repo-maintain-frequency` flag. For example: +kubectl -n velero get secret velero-repo-credentials -o jsonpath='{.data.repository-password}' | base64 -d \ +| kubectl -n velero exec -i deploy/velero -c velero -- \ + restic -r "$PREFIX/default" --cache-dir=/scratch/.cache/restic --password-file=/dev/stdin prune -```bash -kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--default-repo-maintain-frequency=24h0m0s"}]' +kubectl -n velero get secret velero-repo-credentials -o jsonpath='{.data.repository-password}' | base64 -d \ +| kubectl -n velero exec -i deploy/velero -c velero -- \ + restic -r "$PREFIX/kurl" --cache-dir=/scratch/.cache/restic --password-file=/dev/stdin prune ``` -For more information, see [File System Backup](https://velero.io/docs/v1.17/file-system-backup/) in the Velero documentation. +For more information about upgrading Velero, see [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading). ## Additional resources - [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility) - [Troubleshoot Snapshots](/enterprise/snapshots-troubleshooting-backup-restore) - [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading) +- [Configure a Host Path Storage Destination](/enterprise/snapshots-configuring-hostpath) diff --git a/styles/Replicated/Acronyms.yml b/styles/Replicated/Acronyms.yml index 4ab1179f70..c15b2b2c44 100644 --- a/styles/Replicated/Acronyms.yml +++ b/styles/Replicated/Acronyms.yml @@ -65,6 +65,7 @@ exceptions: - URL - USB - UTF + - UID - UUID - VXLAN - XML From 7696c9f385c71a31f2203401de98cb1140acf442 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Mon, 6 Jul 2026 16:37:23 +1200 Subject: [PATCH 11/12] add KOTS release supporting Kopia --- docs/partials/snapshots/_velero-compatibility.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/partials/snapshots/_velero-compatibility.mdx b/docs/partials/snapshots/_velero-compatibility.mdx index bbaa8602b1..4947d1726b 100644 --- a/docs/partials/snapshots/_velero-compatibility.mdx +++ b/docs/partials/snapshots/_velero-compatibility.mdx @@ -5,7 +5,7 @@ The following table lists the versions of Velero that are compatible with each v | KOTS version | Velero version | Supported Kubernetes versions | |------|-------------|-------------| -| TBD and later | 1.11.0 to 1.18.x | 1.33.7, 1.34.1, and 1.35.0 | +| 1.130.7 and later | 1.11.0 to 1.18.x | 1.33.7, 1.34.1, and 1.35.0 | | 1.125.2 and later | 1.11.0 to 1.16.2 | 1.31.4, 1.32.3, and 1.33.0 | | 1.125.0 and later | 1.11.0 to 1.16.1 | 1.31.4, 1.32.3, and 1.33.0 | | 1.124.18 and later | 1.11.0 to 1.16.0 | 1.31.4, 1.32.3, and 1.33.0 | From 2ee17ef7ac4aa381e54df066a92eca26bff6aec8 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Tue, 7 Jul 2026 09:00:29 +1200 Subject: [PATCH 12/12] corrections from review --- .../snapshots-storage-destinations.md | 9 +++------ docs/partials/snapshots/_installVelero.mdx | 19 ++++++++++++++++--- .../snapshots/_velero-compatibility.mdx | 3 ++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/docs/enterprise/snapshots-storage-destinations.md b/docs/enterprise/snapshots-storage-destinations.md index d4046225da..6a3e596449 100644 --- a/docs/enterprise/snapshots-storage-destinations.md +++ b/docs/enterprise/snapshots-storage-destinations.md @@ -35,8 +35,7 @@ To install Velero and configure an AWS storage destination: 1. Run the `velero install` command with these additional flags: * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. - * **Velero 1.10 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. - * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. + * **Velero 1.11 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. **Example:** @@ -61,8 +60,7 @@ To install Velero and configure a GCP storage destination: 1. Run the `velero install` command with these additional flags: * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. - * **Velero 1.10 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. - * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. + * **Velero 1.11 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. **Example:** @@ -86,8 +84,7 @@ To install Velero and configure an Azure storage destination: 1. Run the `velero install` command with these additional flags: * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. - * **Velero 1.10 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. - * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. + * **Velero 1.11 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. **Example:** diff --git a/docs/partials/snapshots/_installVelero.mdx b/docs/partials/snapshots/_installVelero.mdx index a8e7a9f246..1cc078a901 100644 --- a/docs/partials/snapshots/_installVelero.mdx +++ b/docs/partials/snapshots/_installVelero.mdx @@ -1,4 +1,17 @@ -Run one of the following commands to install Velero, depending on the version of the velero CLI you are using: +Run one of the following commands to install Velero, depending on the version of the velero CLI you are using. In the command, replace `$AWS_PLUGIN_VERSION` with the version of the `velero/velero-plugin-for-aws` plugin that is compatible with your Velero version. + +The following table lists the `velero/velero-plugin-for-aws` versions that are compatible with each supported Velero version. For the full Compatibility Matrix, see [Compatibility](https://github.com/velero-io/velero-plugin-for-aws#compatibility) in _velero-plugin-for-aws_. + +| Velero version | AWS plugin version | +|---|---| +| v1.18.x | v1.14.2 | +| v1.17.x | v1.13.x | +| v1.16.x | v1.12.x | +| v1.15.x | v1.11.x | +| v1.14.x | v1.10.x | +| v1.13.x | v1.9.x | +| v1.12.x | v1.8.x | +| v1.11.x | v1.7.x | * **Velero v1.17 and later**: @@ -8,7 +21,7 @@ Run one of the following commands to install Velero, depending on the version of --no-secret \ --use-node-agent \ --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.14.2 + --plugins velero/velero-plugin-for-aws:$AWS_PLUGIN_VERSION ``` * **Velero v1.11 to v1.16**: @@ -19,5 +32,5 @@ Run one of the following commands to install Velero, depending on the version of --no-secret \ --use-node-agent --uploader-type=restic \ --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.14.2 + --plugins velero/velero-plugin-for-aws:$AWS_PLUGIN_VERSION ``` diff --git a/docs/partials/snapshots/_velero-compatibility.mdx b/docs/partials/snapshots/_velero-compatibility.mdx index 4947d1726b..61e5aa4817 100644 --- a/docs/partials/snapshots/_velero-compatibility.mdx +++ b/docs/partials/snapshots/_velero-compatibility.mdx @@ -1,6 +1,7 @@ The following table lists the versions of Velero that are compatible with each version of KOTS, and the Kubernetes versions that each Velero version supports. For the full Velero Compatibility Matrix, see [Velero Compatibility Matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) in the vmware-tanzu/velero repository. -:::note KOTS supports Velero 1.11.0 through 1.18.x. For Velero 1.16 and earlier, KOTS uses Restic to back up pod volume data. For Velero 1.17 and later, KOTS uses Kopia. Existing Restic backups are restorable on Velero 1.17 and 1.18. Velero 1.19 is not supported. +:::note +KOTS supports Velero 1.11.0 through 1.18.x. For Velero 1.16 and earlier, KOTS uses Restic to back up pod volume data. For Velero 1.17 and later, KOTS uses Kopia. Existing Restic backups are restorable on Velero 1.17 and 1.18. Velero 1.19 is not supported. ::: | KOTS version | Velero version | Supported Kubernetes versions |