diff --git a/.gitignore b/.gitignore index d64b3dc3e5..0a8a1fe57d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ yarn-error.log* # Git worktrees .worktrees + +# Local tools / research directories +.tools diff --git a/docs/enterprise/snapshots-config-workflow.md b/docs/enterprise/snapshots-config-workflow.md index 8fcf31b7dc..a937263026 100644 --- a/docs/enterprise/snapshots-config-workflow.md +++ b/docs/enterprise/snapshots-config-workflow.md @@ -22,7 +22,7 @@ To configure snapshots for embedded clusters: - [Configure an NFS Storage Destination](snapshots-configuring-nfs) - [Configure Other Storage Destinations](snapshots-storage-destinations) -1. Optionally increase the default memory for the node-agent (restic) Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). +1. Optionally increase the default memory for the node-agent Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). ## Configuring backup storage for existing clusters @@ -38,7 +38,7 @@ Follow this process to install Velero and configure the snapshots feature: - [Configure an NFS Storage Destination](snapshots-configuring-nfs) - [Configure Other Storage Destinations](snapshots-storage-destinations) -1. Enable access to the Velero namespace if you are using RBAC and optionally increase the default memory for the node-agent (restic) Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). +1. Enable access to the Velero namespace if you are using RBAC and optionally increase the default memory for the node-agent Pod. See [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config). ## Next step diff --git a/docs/enterprise/snapshots-configuring-hostpath.md b/docs/enterprise/snapshots-configuring-hostpath.md index 7225b379cf..23516cc47f 100644 --- a/docs/enterprise/snapshots-configuring-hostpath.md +++ b/docs/enterprise/snapshots-configuring-hostpath.md @@ -1,6 +1,6 @@ import InstallVelero from "../partials/snapshots/_installVelero.mdx" import RegistryCredNote from "../partials/snapshots/_registryCredentialsNote.mdx" -import ResticDaemonSet from "../partials/snapshots/_resticDaemonSet.mdx" +import NodeAgentDaemonSet from "../partials/snapshots/_node-agent-daemonset.mdx" import UpdateDefaultStorage from "../partials/snapshots/_updateDefaultStorage.mdx" import CheckVersion from "../partials/snapshots/_checkVersion.mdx" import KotsAvailability from "../partials/kots/_kots-availability.mdx" @@ -15,6 +15,10 @@ This topic describes how to install Velero and configure a host path as your sto +:::important +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. If you use LVP for HostPath storage, migrate to a Kopia-compatible destination before upgrading to Velero 1.17 or later. Existing LVP backups from Velero 1.16 and earlier are not restorable on Velero 1.17 or later. For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). +::: + ## Requirements * The host path must be a dedicated directory. Do not use a partition used by a service like Docker or Kubernetes for ephemeral storage. @@ -37,7 +41,7 @@ To install Velero and configure host path storage in online environments: 1. -1. +1. 1. Run the following command to configure the host path storage destination: @@ -51,9 +55,9 @@ To install Velero and configure host path storage in online environments: For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. -## Install Velero and configure host path storage in air gapped environments +## Install Velero and configure host path storage in air-gapped environments -To install Velero and configure host path storage in air gapped environments: +To install Velero and configure host path storage in air-gapped environments: 1. @@ -61,7 +65,7 @@ To install Velero and configure host path storage in air gapped environments: -1. +1. 1. Run the following command to configure the host path storage destination: diff --git a/docs/enterprise/snapshots-configuring-nfs.md b/docs/enterprise/snapshots-configuring-nfs.md index 1babc6622d..1cc0b82d33 100644 --- a/docs/enterprise/snapshots-configuring-nfs.md +++ b/docs/enterprise/snapshots-configuring-nfs.md @@ -1,6 +1,6 @@ import InstallVelero from "../partials/snapshots/_installVelero.mdx" import RegistryCredNote from "../partials/snapshots/_registryCredentialsNote.mdx" -import ResticDaemonSet from "../partials/snapshots/_resticDaemonSet.mdx" +import NodeAgentDaemonSet from "../partials/snapshots/_node-agent-daemonset.mdx" import UpdateDefaultStorage from "../partials/snapshots/_updateDefaultStorage.mdx" import CheckVersion from "../partials/snapshots/_checkVersion.mdx" import KotsAvailability from "../partials/kots/_kots-availability.mdx" @@ -15,6 +15,10 @@ This topic describes how to install Velero and configure a Network File System ( +:::important +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. If you use LVP for NFS storage, migrate to a Kopia-compatible destination before upgrading to Velero 1.17 or later. Existing LVP backups from Velero 1.16 and earlier are not restorable on Velero 1.17 or later. For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). +::: + ## Requirements Configuring an NFS server as a snapshots storage destination has the following requirements: @@ -38,7 +42,7 @@ To install Velero and configure NFS storage in an online environment: 1. -1. +1. 1. Run the following command to configure the NFS storage destination: @@ -53,9 +57,9 @@ To install Velero and configure NFS storage in an online environment: For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. -## Install Velero and configure NFS storage in air gapped environments +## Install Velero and configure NFS storage in air-gapped environments -To install Velero and configure NFS storage in air gapped environments: +To install Velero and configure NFS storage in air-gapped environments: 1. @@ -63,7 +67,7 @@ To install Velero and configure NFS storage in air gapped environments: -1. +1. 1. Run the following command to configure the NFS storage destination: diff --git a/docs/enterprise/snapshots-storage-destinations.md b/docs/enterprise/snapshots-storage-destinations.md index 18a9613f9f..6a3e596449 100644 --- a/docs/enterprise/snapshots-storage-destinations.md +++ b/docs/enterprise/snapshots-storage-destinations.md @@ -34,20 +34,20 @@ To install Velero and configure an AWS storage destination: 1. Run the `velero install` command with these additional flags: - * **Velero 1.10 and later**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. - * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. + * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. + * **Velero 1.11 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. **Example:** ``` velero install \ - --provider aws \ - --plugins velero/velero-plugin-for-aws:v1.2.0 \ - --bucket $BUCKET \ - --backup-location-config region=$REGION \ - --secret-file CREDS_FILE \ - --use-node-agent --uploader-type=restic \ - --use-volume-snapshots=false + --provider aws \ + --plugins velero/velero-plugin-for-aws:v1.14.2 \ + --bucket $BUCKET \ + --backup-location-config region=$REGION \ + --secret-file CREDS_FILE \ + --use-node-agent \ + --use-volume-snapshots=false ``` ## Configure GCP storage for online environments @@ -59,18 +59,18 @@ To install Velero and configure a GCP storage destination: 1. Follow the instructions for installing Velero on GCP in the [velero-plugin-for-gcp](https://github.com/vmware-tanzu/velero-plugin-for-gcp#setup) repository in GitHub. 1. Run the `velero install` command with these additional flags: - * **Velero 1.10 and later**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. - * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. + * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. + * **Velero 1.11 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. **Example:** ``` velero install \ --provider gcp \ - --plugins velero/velero-plugin-for-gcp:v1.5.0 \ + --plugins velero/velero-plugin-for-gcp:v1.14.2 \ --bucket $BUCKET \ - --secret-file ./CREDS_FILE - --use-node-agent --uploader-type=restic \ + --secret-file ./CREDS_FILE \ + --use-node-agent \ --use-volume-snapshots=false ``` @@ -83,20 +83,20 @@ To install Velero and configure an Azure storage destination: 1. Follow the instructions for [Install Velero on Azure](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup) in the Velero documentation. 1. Run the `velero install` command with these additional flags: - * **Velero 1.10 and later**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. - * **Velero versions earlier than 1.10**: Use the `--use-restic` and `--use-volume-snapshots=false` flags. + * **Velero 1.17 and later**: Use the `--use-node-agent` and `--use-volume-snapshots=false` flags. + * **Velero 1.11 to 1.16**: Use the `--use-node-agent`, `--uploader-type=restic`, and `--use-volume-snapshots=false` flags. **Example:** ``` velero install \ --provider azure \ - --plugins velero/velero-plugin-for-microsoft-azure:v1.5.0 \ + --plugins velero/velero-plugin-for-microsoft-azure:v1.14.2 \ --bucket $BLOB_CONTAINER \ --secret-file ./CREDS_FILE \ --backup-location-config resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID[,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ - --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] - --use-node-agent --uploader-type=restic \ + --snapshot-location-config apiTimeout=[,resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID] \ + --use-node-agent \ --use-volume-snapshots=false ``` @@ -142,7 +142,7 @@ kubectl kots velero configure-other-s3 \ If no Velero installation is detected, instructions are displayed to install Velero and configure the storage destination. -## Configure s3-compatible storage for air gapped environments +## Configure s3-compatible storage for air-gapped environments > Introduced in Replicated KOTS v1.94.0 @@ -151,7 +151,7 @@ The following S3-compatible object stores are supported for storing backups with - Ceph RADOS v12.2.7 - MinIO -Run the following command to install Velero and configure an S3-compatible storage destination in an air gapped environment. For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. +Run the following command to install Velero and configure an S3-compatible storage destination in an air-gapped environment. For more information about required storage destination flags, see [`velero`](/reference/kots-cli-velero-index) in _Reference_. ```bash kubectl kots velero configure-other-s3 \ diff --git a/docs/enterprise/snapshots-troubleshooting-backup-restore.md b/docs/enterprise/snapshots-troubleshooting-backup-restore.md index 28674dc0d5..75ca6b60f9 100644 --- a/docs/enterprise/snapshots-troubleshooting-backup-restore.md +++ b/docs/enterprise/snapshots-troubleshooting-backup-restore.md @@ -3,7 +3,7 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" # Troubleshoot snapshots -When a snapshot fails, a support bundle will be collected and stored automatically. Because this is a point-in-time collection of all logs and system state at the time of the failed snapshot, this is a good place to view the logs. +When a snapshot fails, KOTS automatically collects and stores a support bundle. This bundle contains all logs and system state at the time of the failure. It is a good place to view the logs. @@ -29,7 +29,7 @@ An error occurred: some backup storage locations are invalid: backup store for l #### Cause -If the cloud access credentials are invalid or do not have access to the location in the configuration, Velero will crashloop. The Velero logs will be included in a support bundle, and the message will look like this. +If the cloud access credentials are invalid or do not have access to the location in the configuration, Velero will crashloop. The support bundle includes the Velero logs, and the message looks like this. #### Solution @@ -53,7 +53,7 @@ An error occurred: some backup storage locations are invalid: backup store for l #### Cause -This error message is caused when Velero is attempting to start, and it is configured to use a reconfigured or re-used bucket. +Velero displays this error message when it attempts to start and uses a reconfigured or re-used bucket. When configuring Velero to use a bucket, the bucket cannot contain other data, or Velero will crash. @@ -78,7 +78,7 @@ time="2023-11-16T21:29:44Z" level=fatal msg="Failed to start metric server for n #### Cause -This is a result of a known issue in Velero 1.12.0 and 1.12.1 where the port is not set correctly when starting the metrics server. This causes the metrics server to fail to start with a `permission denied` error in environments that do not run MinIO and have Host Path, NFS, or internal storage destinations configured. When the metrics server fails to start, the node-agent Pod crashes. For more information about this issue, see [the GitHub issue details](https://github.com/vmware-tanzu/velero/issues/6792). +This issue occurs in Velero 1.12.0 and 1.12.1. Velero does not set the port correctly when starting the metrics server. The metrics server fails to start with a `permission denied` error. The error occurs in environments that do not run MinIO and have Host Path, Network File System (NFS), or internal storage destinations configured. When the metrics server fails to start, the node-agent Pod crashes. For more information about this issue, see [the GitHub issue details](https://github.com/vmware-tanzu/velero/issues/6792). #### Solution @@ -99,27 +99,29 @@ timed out after 12h0m0s #### Cause -This error message appears when the node-agent (restic) Pod operation timeout limit is reached. In Velero v1.4.2 and later, the default timeout is 240 minutes. +This error message appears when the node-agent Pod operation reaches the timeout limit. The default timeout is 240 minutes. -Restic is an open-source backup tool. Velero integrates with Restic to provide a solution for backing up and restoring Kubernetes volumes. For more information about the Velero Restic integration, see [File System Backup](https://velero.io/docs/v1.10/file-system-backup/) in the Velero documentation. +For Velero 1.16 and earlier, Velero integrates with Restic to provide a solution for backing up and restoring Kubernetes volumes. For more information, see [File System Backup](https://velero.io/docs/v1.10/file-system-backup/) in the Velero documentation. + +For Velero 1.17 and later, Velero uses Kopia for file-system backups by default. #### Solution Use the kubectl Kubernetes command-line tool to patch the Velero deployment to increase the timeout: -**Velero version 1.10 and later**: +**Velero 1.17 and later**: ```bash kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--fs-backup-timeout=TIMEOUT_LIMIT"}]' ``` -**Velero versions less than 1.10**: +**Velero 1.16 and earlier**: ```bash kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--restic-timeout=TIMEOUT_LIMIT"}]' ``` -Replace `TIMEOUT_LIMIT` with a length of time for the node-agent (restic) Pod operation timeout in hours, minutes, and seconds. Use the format `0h0m0s`. For example, `48h30m0s`. +Replace `TIMEOUT_LIMIT` with a length of time for the node-agent Pod operation timeout in hours, minutes, and seconds. Use the format `0h0m0s`. For example, `48h30m0s`. :::note The timeout value reverts back to the default value if you rerun the `velero install` command. @@ -129,7 +131,7 @@ The timeout value reverts back to the default value if you rerun the `velero ins #### Symptom -The node-agent (restic) Pod is killed by the Linux kernel Out Of Memory (OOM) killer or snapshots are failing with errors simlar to: +The Linux kernel Out Of Memory (OOM) killer kills the node-agent Pod, or snapshots fail with errors similar to: ``` pod volume backup failed: ... signal: killed @@ -137,15 +139,15 @@ pod volume backup failed: ... signal: killed #### Cause -Velero sets default limits for the velero Pod and the node-agent (restic) Pod during installation. There is a known issue with Restic that causes high memory usage, which can result in failures during snapshot creation when the Pod reaches the memory limit. +Velero sets default limits for the velero Pod and the node-agent Pod during installation. For Velero 1.16 and earlier, a known Restic issue causes high memory usage. This high memory usage can cause failures during snapshot creation when the Pod reaches the memory limit. For Velero 1.17 and later, Velero uses Kopia for file-system backups by default. Large volumes with Kopia can also require a higher memory limit. -For more information, see the [Restic backup — OOM-killed on raspberry pi after backing up another computer to same repo](https://github.com/restic/restic/issues/1988) issue in the restic GitHub repository. +For more information about the Restic issue, see the [Restic backup — OOM-killed on raspberry pi after backing up another computer to same repo](https://github.com/restic/restic/issues/1988) issue in the Restic GitHub repository. #### Solution -### At least one source file could not be read +### Velero cannot read at least one source file #### Symptom @@ -157,7 +159,7 @@ Error backing up item...Warning: at least one source file could not be read #### Cause -There are file changes between Restic's initial scan of the volume and during the backup to Restic store. +For Velero 1.16 and earlier, there are file changes between Restic's initial scan of the volume and during the backup to the Restic store. #### Solution @@ -167,64 +169,104 @@ To resolve this issue, do one of the following: * Freeze the file system to ensure all pending disk I/O operations have completed prior to taking a snapshot. For more information, see [Hook Example with fsfreeze](https://velero.io/docs/main/backup-hooks/#hook-example-with-fsfreeze) in the Velero documentation. -## Snapshot restore is failing +## Kopia file-system backup issues (Velero 1.17 and later) -### Service nodeport is already allocated +### Data mover pods do not start or complete -#### Symptom +#### Cause -In the Replicated KOTS Admin Console, you see an **Application failed to restore** error message that indicates the port number for a static NodePort is already in use. For example: +For Velero 1.17 and later, Kopia spawns data mover pods from the node-agent. If a backup or restore stays in progress, the data mover pods might not start or complete. -![Snapshot Troubleshoot Service NodePort](/images/snapshot-troubleshoot-service-nodeport.png) +#### Solution -[View a larger version of this image](/images/snapshot-troubleshoot-service-nodeport.png) +Check the node-agent logs for errors that prevent data mover pods from starting. Verify that the cluster can pull the data mover pod image and that any pod security policies or security context constraints allow the pod to start. + +### BackupRepository is not available #### Cause -There is a known issue in Kubernetes versions earlier than version 1.19 where using a static NodePort for services can collide in multi-primary high availability setups when recreating the services. For more information about this known issue, see https://github.com/kubernetes/kubernetes/issues/85894. +For Velero 1.17 and later, Kopia uses `BackupRepository` custom resources (CRs) to manage repositories. A backup or restore can fail if the `BackupRepository` CR is not available or is in a failed state. #### Solution -This issue is fixed in Kubernetes version 1.19. To resolve this issue, upgrade to Kubernetes version 1.19 or later. +Check the status of the `BackupRepository` CRs: -For more infromation about the fix, see https://github.com/kubernetes/kubernetes/pull/89937. +```bash +kubectl get backuprepositories -n velero +``` -### Partial snapshot restore is stuck in progress +Describe any CR that is not in a Ready state to view the error: -#### Symptom +```bash +kubectl describe backuprepository BACKUP_REPOSITORY_NAME -n velero +``` -In the Admin Console, you see at least one volume restore progress bar frozen at 0%. Example Admin Console display: +Common errors include invalid credentials, network connectivity issues, or problems with the underlying storage. After you resolve the issue, Velero retries the backup repository operations. -![Snapshot Troubleshoot Frozen Restore](/images/snapshot-troubleshoot-frozen-restore.png) +### Read-only root filesystem errors -You can confirm this is the same issue by running `kubectl get pods -n `, and you should see at least one pod stuck in initialization: +#### Cause -```shell -NAME READY STATUS RESTARTS AGE -example-mysql-0 0/1 Init:0/2 0 4m15s #<- the offending pod -example-nginx-77b878b4f-zwv2h 3/3 Running 0 4m15s -``` +For Velero 1.17 and later, Kopia needs writable directories for cache and configuration. The default paths are `/home/cnb/udmrepo` and `/home/cnb/.cache`. If `ReadOnlyRootFilesystem` applies to the Velero or node-agent pods, Kopia cannot write to these directories and the backup or restore fails. + +#### Solution + +Add `emptyDir` volumes for `/home/cnb/udmrepo` and `/home/cnb/.cache` to the Velero deployment and the node-agent daemon set. Mount the volumes at the required paths so Kopia can write cache and configuration data. + +### LVP storage location is unavailable after upgrade + +#### Cause + +The Local Volume Provider (LVP) is not compatible with Kopia. If you upgrade to Velero 1.17 or later and the existing storage location uses LVP, snapshots fail because Kopia cannot write to LVP storage. + +:::important +LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Before you upgrade, migrate to a Kopia-compatible storage destination. For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). +::: + +#### Solution + +Before you upgrade to Velero 1.17 or later, migrate from LVP to a Kopia-compatible destination. Replicated recommends one of the following options: + +* Reinstall KOTS with `--with-minio=true`. +* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Install the target Velero plugin before you reconfigure the storage location. + +For more information, see [Upgrade Velero for snapshots](snapshots-velero-upgrading). + + +## Snapshot restore is failing + +### Service NodePort is already allocated + +#### Symptom + +In the Replicated KOTS Admin Console, you see an **Application failed to restore** error. The error indicates that the port number for a static NodePort is already in use. For example: + +![Snapshot Troubleshoot Service NodePort](/images/snapshot-troubleshoot-service-nodeport.png) + +[View a larger version of this image](/images/snapshot-troubleshoot-service-nodeport.png) #### Cause -We have seen this issue with Velero version 1.5.4 and opened up this issue with the project to inspect the root cause: https://github.com/vmware-tanzu/velero/issues/3686. However we have not experienced this using Velero 1.6.0 or later. +A known issue in Kubernetes versions earlier than version 1.19 can cause static NodePort services to collide in multi-primary high availability setups. This collision occurs when recreating the services. For more information about this known issue, see https://github.com/kubernetes/kubernetes/issues/85894. #### Solution -Upgrade Velero to 1.9.0. You can upgrade using Replicated kURL. Or, to follow the Velero upgrade instructions, see [Upgrading to Velero 1.9](https://velero.io/docs/v1.9/upgrade-to-1.9/) in the Velero documentation. +Kubernetes version 1.19 fixes this issue. To resolve this issue, upgrade to Kubernetes version 1.19 or later. + +For more information about the fix, see https://github.com/kubernetes/kubernetes/pull/89937. ### Partial snapshot restore finishes with warnings #### Symptom -In the Admin Console, when the partial snapshot restore completes, you see warnings indicating that Endpoint resources were not restored: +In the Admin Console, when the partial snapshot restore completes, you see warnings indicating that Velero did not restore Endpoint resources: ![Snapshot Troubleshoot Restore Warnings](/images/snapshot-troubleshoot-restore-warnings.png) #### Cause -The resource restore priority was changed in Velero 1.10.3 and 1.11.0, which leads to this warning when restoring Endpoint resources. For more information about this issue, see [the issue details](https://github.com/vmware-tanzu/velero/issues/6280) in GitHub. +Velero changed the resource restore priority in 1.10.3 and 1.11.0, which leads to this warning when restoring Endpoint resources. For more information about this issue, see [the issue details](https://github.com/vmware-tanzu/velero/issues/6280) in GitHub. #### Solution -These warnings do not necessarily mean that the restore itself failed. The endpoints likely do exist as they are created by Kubernetes when the related Service resources were restored. However, to prevent encountering these warnings, use Velero version 1.11.1 or later. +These warnings do not necessarily mean that the restore itself failed. The endpoints likely exist because Kubernetes creates them when the restore process restores the related Service resources. However, to prevent encountering these warnings, use Velero version 1.11.1 or later. diff --git a/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx b/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx new file mode 100644 index 0000000000..e0f037a0fe --- /dev/null +++ b/docs/enterprise/snapshots-troubleshooting-reclaim-space.mdx @@ -0,0 +1,78 @@ +import KotsAvailability from "../partials/kots/_kots-availability.mdx" + +# Deleted snapshots do not reclaim disk space on host path storage + +This topic describes why deleted snapshots might not reclaim disk space on embedded kURL clusters. It also explains how to reclaim the space manually. + +The issue applies to clusters that use Host Path storage with the Velero Restic uploader. + + + +## Symptom + +On an embedded kURL cluster that uses Host Path storage with the Velero Restic uploader, disk space is not reclaimed after you delete snapshots. You can delete snapshots from the Admin Console or with the KOTS CLI. The disk continues to fill even though only the retained backups appear in `velero get backup`. + +## Cause + +Velero's repository maintenance job runs a `restic prune` operation. The prune removes data that is no longer referenced from the Restic repository. This frees the disk space that deleted or expired snapshots used. + +On Velero versions earlier than 1.17.0, the repository maintenance job does not inherit the Velero Pod security context. The job runs as a different user than the node-agent Pod that created the Restic repository files on the Host Path store. The Restic files belong to UID 1001 with mode 0700. The maintenance job receives a `permission denied` error when it tries to read the repository. The prune operation then fails. + +Because the prune fails, data from deleted or expired snapshots is never reclaimed. The backup disk grows without bound. + +For example, the maintenance Pod logs show an error similar to the following: + +``` +restic prune --repo=.../restic/default ... +Fatal: unable to open repository at .../restic/default: + ReadDir: open .../restic/default/keys: permission denied +``` + +Velero 1.17.0 and later include the upstream fix. The fix copies the security context from the origin Pod to the maintenance job. The automated prune operation can then read the repository files and reclaim disk space. For more information, see [Copy security context from origin pod](https://github.com/vmware-tanzu/velero/pull/8943) in the Velero repository. + +## Solution + +The solution depends on the Velero version. + +### Velero 1.17.0 and later + +Upgrade to Velero 1.17.0 or later. The automated Restic repository maintenance job can then read the repository files and reclaim disk space. After the upgrade, you can adjust the maintenance frequency with the `--default-repo-maintain-frequency` Velero server flag. For example: + +```bash +kubectl patch deployment velero -n velero --type json -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--default-repo-maintain-frequency=48h0m0s"}]' +``` + +Replace `48h0m0s` with the desired frequency. Velero uses the default value if you do not set the flag. + +:::note +The `--default-restic-prune-frequency` flag was the previous name for this flag. Velero removed `--default-restic-prune-frequency` in version 1.10.0. It renamed the flag to `--default-repo-maintain-frequency`. On Velero 1.10.0 and later, `--default-restic-prune-frequency` is not a recognized flag and has no effect. For more information, see the [Velero 1.10 breaking changes](https://github.com/vmware-tanzu/velero/blob/main/changelogs/CHANGELOG-1.10.md#breaking-changes). +::: + +For more information about file-system backups, see [File System Backup](https://velero.io/docs/v1.17/file-system-backup/) in the Velero documentation. + +### Velero 1.16.x and earlier + +On Velero versions earlier than 1.17.0, the automated prune fails regardless of the maintenance frequency. Tuning the frequency does not help. To reclaim disk space until you can upgrade to Velero 1.17.0 or later, run `restic prune` manually from inside the Velero Pod. The Velero Pod runs as the correct user (UID 1001), so the prune operation can read the repository files. + +Run the following commands to prune both the `default` and `kurl` Restic repositories: + +```bash +PREFIX=$(kubectl -n velero get bsl default -o jsonpath='{.spec.config.resticRepoPrefix}') + +kubectl -n velero get secret velero-repo-credentials -o jsonpath='{.data.repository-password}' | base64 -d \ +| kubectl -n velero exec -i deploy/velero -c velero -- \ + restic -r "$PREFIX/default" --cache-dir=/scratch/.cache/restic --password-file=/dev/stdin prune + +kubectl -n velero get secret velero-repo-credentials -o jsonpath='{.data.repository-password}' | base64 -d \ +| kubectl -n velero exec -i deploy/velero -c velero -- \ + restic -r "$PREFIX/kurl" --cache-dir=/scratch/.cache/restic --password-file=/dev/stdin prune +``` + +For more information about upgrading Velero, see [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading). + +## Additional resources + +- [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility) +- [Troubleshoot Snapshots](/enterprise/snapshots-troubleshooting-backup-restore) +- [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading) +- [Configure a Host Path Storage Destination](/enterprise/snapshots-configuring-hostpath) diff --git a/docs/enterprise/snapshots-velero-cli-installing.md b/docs/enterprise/snapshots-velero-cli-installing.md index 891a917341..96e5cc6f6e 100644 --- a/docs/enterprise/snapshots-velero-cli-installing.md +++ b/docs/enterprise/snapshots-velero-cli-installing.md @@ -5,11 +5,15 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" You install the Velero CLI before installing Velero and configuring a storage destination for backups. :::note -For embedded clusters created with Replicated kURL, if the kURL Installer spec included the Velero add-on, then Velero was automatically installed with default internal storage. Replicated recommends that you proceed to change the default internal storage because it is insufficient for disaster recovery. See [Updating Storage Settings in the Admin Console](snapshots-updating-with-admin-console). +For embedded clusters created with Replicated kURL, check whether the kURL Installer spec included the Velero add-on. If so, the kURL installer automatically installed Velero with default internal storage. Replicated recommends that you change the default internal storage because it is insufficient for disaster recovery. See [Updating Storage Settings in the Admin Console](snapshots-updating-with-admin-console). ::: +:::note +Velero 1.17 and later uses Kopia as the default file-system backup uploader. For more information, see [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). +::: + ## Install the Velero CLI in an online cluster To install the Velero CLI in an online cluster: @@ -19,7 +23,7 @@ To install the Velero CLI in an online cluster: - (Embedded kURL cluster) Run an SSH command to access and authenticate to your cluster node. - (Existing cluster) Open a terminal in the environment that you manage the cluster from, which can be a local machine that has kubectl installed. -1. Check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although earlier versions of Velero are supported, Replicated recommends using the latest supported version. For more information about supported versions, see [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). +1. Check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although Replicated supports earlier versions of Velero, Replicated recommends using the latest supported version. For more information about supported versions, see [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). Note the version number for the next step. @@ -34,10 +38,10 @@ To install the Velero CLI in an online cluster: **Example:** ``` - curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.10.1/velero-v1.10.1-linux-amd64.tar.gz + curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.18.2/velero-v1.18.2-linux-amd64.tar.gz ``` -1. Run the following command to uncompress the TAR file: +1. Run the following command to extract the TAR file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz @@ -55,11 +59,11 @@ To install the Velero CLI in an online cluster: You might get an error message stating that there are no matches for the server version. This is acceptable, as long as you get a confirmation for the client version. After the Velero installation, you also see the server version. -## Install the Velero CLI in an air gapped cluster +## Install the Velero CLI in an air-gapped cluster -To install the Velero CLI in an air gapped cluster: +To install the Velero CLI in an air-gapped cluster: -1. From a computer with internet access, check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although earlier versions of Velero are supported, Replicated recommends using the latest supported version. See [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). +1. From a computer with internet access, check for the latest supported release of the Velero CLI for **Linux AMD64** in the Velero GitHub repo at https://github.com/vmware-tanzu/velero/releases. Although Replicated supports earlier versions of Velero, Replicated recommends using the latest supported version. See [Velero Version Compatibility](/vendor/snapshots-overview#velero-version-compatibility). Note the version number for the next step. @@ -74,12 +78,12 @@ To install the Velero CLI in an air gapped cluster: **Example:** ``` - curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.10.1/velero-v1.10.1-linux-amd64.tar.gz + curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.18.2/velero-v1.18.2-linux-amd64.tar.gz ``` -1. Copy the TAR file to the air gapped node. +1. Copy the TAR file to the air-gapped node. -1. Run the following command to uncompress the TAR file: +1. Run the following command to extract the TAR file: ``` tar zxvf velero-VERSION-linuxamd64.tar.gz diff --git a/docs/enterprise/snapshots-velero-installing-config.mdx b/docs/enterprise/snapshots-velero-installing-config.mdx index f8fa92925a..168c123bf9 100644 --- a/docs/enterprise/snapshots-velero-installing-config.mdx +++ b/docs/enterprise/snapshots-velero-installing-config.mdx @@ -11,7 +11,7 @@ This topic describes how to configure namespace access and the memory limit for ## Overview -The Replicated KOTS Admin Console requires access to the namespace where Velero is installed. If your Admin Console is running with minimal role-based-access-control (RBAC) privileges, you must enable the Admin Console to access Velero. +The Replicated KOTS Admin Console requires access to the namespace where Velero runs. If your Admin Console is running with minimal role-based-access-control (RBAC) privileges, you must enable the Admin Console to access Velero. Additionally, if the application uses a large amount of memory, you can configure the default memory limit to help ensure that Velero runs successfully with snapshots. @@ -26,7 +26,7 @@ kubectl kots velero ensure-permissions --namespace ADMIN_CONSOLE_NAMESPACE --vel ``` Replace: * `ADMIN_CONSOLE_NAMESPACE` with the namespace on the cluster where the Admin Console is running. -* `VELERO_NAMESPACE` with the namespace on the cluster where Velero is installed. +* `VELERO_NAMESPACE` with the namespace on the cluster where Velero runs. For more information, see [`velero ensure-permissions`](/reference/kots-cli-velero-ensure-permissions/) in the KOTS CLI documentation. For more information about RBAC privileges for the Admin Console, see [Kubernetes RBAC](/vendor/packaging-rbac). @@ -34,7 +34,7 @@ For more information, see [`velero ensure-permissions`](/reference/kots-cli-vele This section applies to all online and air gap installations. -Velero sets default limits for the velero Pod and the node-agent (restic) Pod during installation. There is a known issue with restic that causes high memory usage, which can result in failures during backup creation when the Pod reaches the memory limit. +Velero sets default limits for the velero Pod and the node-agent Pod during installation. There is a known issue with the file-system backup uploader. High memory usage from this issue can cause failures during backup creation when the Pod reaches the memory limit. diff --git a/docs/enterprise/snapshots-velero-upgrading.mdx b/docs/enterprise/snapshots-velero-upgrading.mdx new file mode 100644 index 0000000000..e1bcd55e9a --- /dev/null +++ b/docs/enterprise/snapshots-velero-upgrading.mdx @@ -0,0 +1,92 @@ +import KotsAvailability from "../partials/kots/_kots-availability.mdx" + +# Upgrade Velero for snapshots + +This topic describes how to upgrade Velero for Replicated snapshots. + + + +## Overview + +Velero 1.17 and later uses Kopia as the default uploader for file-system backups. If you upgrade from Velero 1.16 or earlier, verify that your storage destination is compatible with Kopia before you upgrade. Some storage destinations, such as the Local Volume Provider (LVP), are not compatible with Kopia. Migrate these destinations before the upgrade. + +For more information about snapshot troubleshooting, see [Troubleshoot Snapshots](snapshots-troubleshooting-backup-restore). + +## Before you upgrade + +Complete the following prerequisites before you upgrade Velero: + +1. Check the current storage destination type. If the storage location uses LVP, migrate to a Kopia-compatible destination before you upgrade. For more information, see [Migrate from Local Volume Provider (LVP) to a Kopia-compatible destination](#migrate-from-local-volume-provider-lvp-to-a-kopia-compatible-destination). + +1. Take a snapshot of the application before you upgrade. This snapshot lets you restore the application if the upgrade fails. + +1. Install the Velero plugin for the target storage destination, if needed. For example, verify that the plugin image is available in the registry for air-gapped environments. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). + +## Migrate from local volume provider (LVP) to a Kopia-compatible destination + +LVP is not compatible with Kopia. If the storage location uses LVP, migrate to a Kopia-compatible destination before you upgrade to Velero 1.17 or later. + +:::important +LVP backups created on Velero 1.16 and earlier are not restorable on Velero 1.17 and later. Replicated recommends that you migrate to a Kopia-compatible destination before you upgrade. +::: + +In-place reconfiguration from LVP to a filesystem-based Minio deployment is not supported. The `minio-enabled-snapshots=false` setting persists, so reconfiguring to filesystem Minio in the Admin Console does not enable a Kopia-compatible object store. + +Replicated recommends one of the following options: + +* Reinstall KOTS with `--with-minio=true`. This installs a Minio object store that is compatible with Kopia. +* Reconfigure the storage location to use an external S3-compatible object store, such as Amazon S3, Google Cloud Storage, Azure Blob Storage, or another S3-compatible provider. Install the target Velero plugin before you reconfigure the storage location. For more information, see [Configure Other Storage Destinations](snapshots-storage-destinations). + +For more information about LVP storage issues after upgrade, see [LVP storage location is unavailable after upgrade](snapshots-troubleshooting-backup-restore#lvp-storage-location-is-unavailable-after-upgrade) in _Troubleshoot Snapshots_. + +## Upgrade one version at a time + +Replicated recommends that you upgrade Velero one version at a time. For example, if you are running Velero 1.15, upgrade to Velero 1.16 first, verify that snapshots work, and then upgrade to Velero 1.17. Upgrading incrementally reduces the risk of incompatible custom resources or storage locations. + +To upgrade Velero, follow the upgrade instructions for the target Velero version in the [Velero documentation](https://velero.io/docs/). KOTS does not provide an Admin Console button or a KOTS CLI command to upgrade Velero. The cluster administrator performs the upgrade manually. + +## Verify the upgrade + +After you upgrade Velero, verify that the upgrade is successful: + +1. Run `velero version` to confirm the expected Velero version. + +1. Check the status of the `BackupStorageLocation`: + + ```bash + velero backup-location get + ``` + + The BackupStorageLocation must be in an `Available` state before you create backups. If it is not available, describe the BackupStorageLocation to view the error: + + ```bash + kubectl describe backupstoragelocation -n velero + ``` + +1. Check the status of the `BackupRepository` CRs: + + ```bash + kubectl get backuprepositories -n velero + ``` + + All `BackupRepository` CRs must be in a `Ready` state before you create backups. If a CR is not ready, describe the CR to view the error: + + ```bash + kubectl describe backuprepository -n velero + ``` + +1. Create a test backup and verify that it completes successfully. For more information, see [Create and Schedule Backups](snapshots-creating). + +## Restore existing Restic backups + +Velero 1.17 and 1.18 can restore Restic backups created on Velero 1.16 and earlier. To restore an existing Restic backup, use the same storage location and credentials that you used to create the backup. + +## Additional resources + +* [Troubleshoot Snapshots](snapshots-troubleshooting-backup-restore) +* [Configure Other Storage Destinations](snapshots-storage-destinations) +* [Configure a Host Path Storage Destination](snapshots-configuring-hostpath) +* [Configure an NFS Storage Destination](snapshots-configuring-nfs) +* [Configure Namespace Access and Memory Limit](snapshots-velero-installing-config) +* [Create and Schedule Backups](snapshots-creating) +* [Update storage settings](snapshots-updating-with-admin-console) diff --git a/docs/partials/snapshots/_installVelero.mdx b/docs/partials/snapshots/_installVelero.mdx index cdd03f7aa4..1cc078a901 100644 --- a/docs/partials/snapshots/_installVelero.mdx +++ b/docs/partials/snapshots/_installVelero.mdx @@ -1,23 +1,36 @@ -Run one of the following commands to install Velero, depending on the version of the velero CLI you are using: +Run one of the following commands to install Velero, depending on the version of the velero CLI you are using. In the command, replace `$AWS_PLUGIN_VERSION` with the version of the `velero/velero-plugin-for-aws` plugin that is compatible with your Velero version. - * **Velero v1.10 and later**: +The following table lists the `velero/velero-plugin-for-aws` versions that are compatible with each supported Velero version. For the full Compatibility Matrix, see [Compatibility](https://github.com/velero-io/velero-plugin-for-aws#compatibility) in _velero-plugin-for-aws_. + +| Velero version | AWS plugin version | +|---|---| +| v1.18.x | v1.14.2 | +| v1.17.x | v1.13.x | +| v1.16.x | v1.12.x | +| v1.15.x | v1.11.x | +| v1.14.x | v1.10.x | +| v1.13.x | v1.9.x | +| v1.12.x | v1.8.x | +| v1.11.x | v1.7.x | + + * **Velero v1.17 and later**: ```bash velero install \ --no-default-backup-location \ --no-secret \ - --use-node-agent --uploader-type=restic \ + --use-node-agent \ --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.5.3 + --plugins velero/velero-plugin-for-aws:$AWS_PLUGIN_VERSION ``` - * **Velero versions earlier than v1.10**: + * **Velero v1.11 to v1.16**: ```bash velero install \ --no-default-backup-location \ --no-secret \ - --use-restic \ + --use-node-agent --uploader-type=restic \ --use-volume-snapshots=false \ - --plugins velero/velero-plugin-for-aws:v1.5.3 - ``` \ No newline at end of file + --plugins velero/velero-plugin-for-aws:$AWS_PLUGIN_VERSION + ``` diff --git a/docs/partials/snapshots/_node-agent-daemonset.mdx b/docs/partials/snapshots/_node-agent-daemonset.mdx new file mode 100644 index 0000000000..236d375f88 --- /dev/null +++ b/docs/partials/snapshots/_node-agent-daemonset.mdx @@ -0,0 +1,7 @@ +Configure the node-agent DaemonSet specification if your cluster uses one of the following Kubernetes distributions: + * RancherOS + * OpenShift + * Microsoft Azure + * VMware Tanzu Kubernetes Grid Integrated Edition + +For information about how to configure the node-agent DaemonSet for these distributions, see [Configure node-agent DaemonSet spec](https://velero.io/docs/v1.17/file-system-backup/#configure-node-agent-daemonset-spec) in the Velero documentation. \ No newline at end of file diff --git a/docs/partials/snapshots/_node-agent-mem-limit.mdx b/docs/partials/snapshots/_node-agent-mem-limit.mdx index 1f66e903b4..0c25dc384e 100644 --- a/docs/partials/snapshots/_node-agent-mem-limit.mdx +++ b/docs/partials/snapshots/_node-agent-mem-limit.mdx @@ -1,29 +1,13 @@ -Increase the default memory limit for the node-agent (restic) Pod if your application is particularly large. For more information about configuring Velero resource requests and limits, see [Customize resource requests and limits](https://velero.io/docs/v1.10/customize-installation/#customize-resource-requests-and-limits) in the Velero documentation. +Increase the default memory limit for the node-agent Pod if your application is particularly large. For more information about configuring Velero resource requests and limits, see [Customize resource requests and limits](https://velero.io/docs/v1.17/customize-installation/#customize-resource-requests-and-limits) in the Velero documentation. -For example, the following kubectl commands will increase the memory limit for the node-agent (restic) daemon set from the default of 1Gi to 2Gi. - -**Velero 1.10 and later**: +For example, the following kubectl command increases the memory limit for the node-agent DaemonSet from the default of 1Gi to 2Gi: ``` kubectl -n velero patch daemonset node-agent -p '{"spec":{"template":{"spec":{"containers":[{"name":"node-agent","resources":{"limits":{"memory":"2Gi"}}}]}}}}' ``` -**Velero versions earlier than 1.10**: - -``` -kubectl -n velero patch daemonset restic -p '{"spec":{"template":{"spec":{"containers":[{"name":"restic","resources":{"limits":{"memory":"2Gi"}}}]}}}}' -``` - -Alternatively, you can potentially avoid the node-agent (restic) Pod reaching the memory limit during snapshot creation by running the following kubectl command to lower the memory garbage collection target percentage on the node-agent (restic) daemon set: - -**Velero 1.10 and later**: +Alternatively, you can lower the memory garbage collection target percentage on the node-agent DaemonSet. This can help the node-agent Pod avoid reaching the memory limit during snapshot creation. Run the following kubectl command: ``` kubectl -n velero set env daemonset/node-agent GOGC=1 -``` - -**Velero versions earlier than 1.10**: - -``` -kubectl -n velero set env daemonset/restic GOGC=1 ``` \ No newline at end of file diff --git a/docs/partials/snapshots/_registryCredentialsNote.mdx b/docs/partials/snapshots/_registryCredentialsNote.mdx index fbab6aaeda..bf29501df2 100644 --- a/docs/partials/snapshots/_registryCredentialsNote.mdx +++ b/docs/partials/snapshots/_registryCredentialsNote.mdx @@ -1,5 +1,5 @@ :::note -It is typical for the velero and node-agent (restic) Pods to be in the `ErrImagePull` or `ImagePullBackOff` state after you run the `velero install` command because Velero does not support passing registry credentials during installation. In Replicated KOTS v1.94.0 and later, this situation resolves itself after you complete the instructions to configure the storage destination. +It is typical for the velero and node-agent Pods to be in the `ErrImagePull` or `ImagePullBackOff` state after you run the `velero install` command because Velero does not support passing registry credentials during installation. In Replicated KOTS v1.94.0 and later, this situation resolves itself after you complete the instructions to configure the storage destination. If you are on an earlier version of KOTS, Replicated recommends that you upgrade to KOTS v1.94.0 or later. Otherwise, you must patch the Velero deployment manually and add the image pull secret to access the registry. ::: \ No newline at end of file diff --git a/docs/partials/snapshots/_resticDaemonSet.mdx b/docs/partials/snapshots/_resticDaemonSet.mdx deleted file mode 100644 index b54778e92b..0000000000 --- a/docs/partials/snapshots/_resticDaemonSet.mdx +++ /dev/null @@ -1,7 +0,0 @@ -Configure the Restic DaemonSet specification if your cluster uses one of the following Kubernetes distributions: - * RancherOS - * OpenShift - * Microsoft Azure - * VMware Tanzu Kubernetes Grid Integrated Edition - -For information about how to configure the Restic DaemonSet for these distributions, see [Configure Restic DaemonSet spec](https://velero.io/docs/v1.9/restic/#configure-restic-daemonset-spec) in the Velero documentation. \ No newline at end of file diff --git a/docs/partials/snapshots/_velero-compatibility.mdx b/docs/partials/snapshots/_velero-compatibility.mdx index c546279f61..61e5aa4817 100644 --- a/docs/partials/snapshots/_velero-compatibility.mdx +++ b/docs/partials/snapshots/_velero-compatibility.mdx @@ -1,13 +1,15 @@ -The following table lists which versions of Velero are compatible with each version of KOTS. For a list of the supported Kubernetes versions for each Velero version, see [Velero compatibility matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) in the vmware-tanzu/velero repository. +The following table lists the versions of Velero that are compatible with each version of KOTS, and the Kubernetes versions that each Velero version supports. For the full Velero Compatibility Matrix, see [Velero Compatibility Matrix](https://github.com/vmware-tanzu/velero#velero-compatibility-matrix) in the vmware-tanzu/velero repository. -:::note If you're running KOTS with Velero 1.17+, pod volume backups using Restic are unavailable. This is because Velero 1.17 removed the `--uploader-type=restic` flag. Restores of existing Restic backups still work. If you depend on snapshots, stay on Velero 1.16.x until a KOTS update with Kopia support ships. +:::note +KOTS supports Velero 1.11.0 through 1.18.x. For Velero 1.16 and earlier, KOTS uses Restic to back up pod volume data. For Velero 1.17 and later, KOTS uses Kopia. Existing Restic backups are restorable on Velero 1.17 and 1.18. Velero 1.19 is not supported. ::: -| KOTS version | Velero version | -|------|-------------| -| 1.125.2 and later | 1.11.0 to 1.16.2 | -| 1.125.0 and later | 1.11.0 to 1.16.1 | -| 1.124.18 and later | 1.11.0 to 1.16.0 | -| 1.124.0 and later | 1.11.0 to 1.15.2 | -| 1.123.1 and later | 1.11.0 to 1.15.1 | -| 1.117.3 and later | 1.11.0 to 1.14.1 | +| KOTS version | Velero version | Supported Kubernetes versions | +|------|-------------|-------------| +| 1.130.7 and later | 1.11.0 to 1.18.x | 1.33.7, 1.34.1, and 1.35.0 | +| 1.125.2 and later | 1.11.0 to 1.16.2 | 1.31.4, 1.32.3, and 1.33.0 | +| 1.125.0 and later | 1.11.0 to 1.16.1 | 1.31.4, 1.32.3, and 1.33.0 | +| 1.124.18 and later | 1.11.0 to 1.16.0 | 1.31.4, 1.32.3, and 1.33.0 | +| 1.124.0 and later | 1.11.0 to 1.15.2 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 | +| 1.123.1 and later | 1.11.0 to 1.15.1 | 1.28.8, 1.29.8, 1.30.4, and 1.31.1 | +| 1.117.3 and later | 1.11.0 to 1.14.1 | 1.27.9, 1.28.9, and 1.29.4 | diff --git a/docs/reference/kots-cli-velero-configure-hostpath.mdx b/docs/reference/kots-cli-velero-configure-hostpath.mdx index 97d4b76d53..2f545fec0c 100644 --- a/docs/reference/kots-cli-velero-configure-hostpath.mdx +++ b/docs/reference/kots-cli-velero-configure-hostpath.mdx @@ -8,6 +8,10 @@ import Help from "../partials/kots-cli/_help.mdx" Configure snapshots to use a host path as storage destination. +:::note +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. By default, KOTS uses the S3-compatible filesystem MinIO path for HostPath storage when MinIO is enabled. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. For more information, see [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading). +::: + ### Usage ```bash diff --git a/docs/reference/kots-cli-velero-configure-nfs.mdx b/docs/reference/kots-cli-velero-configure-nfs.mdx index 11405c3ddd..1616226d4e 100644 --- a/docs/reference/kots-cli-velero-configure-nfs.mdx +++ b/docs/reference/kots-cli-velero-configure-nfs.mdx @@ -8,6 +8,10 @@ import Help from "../partials/kots-cli/_help.mdx" Configures snapshots to use NFS as storage destination. +:::note +The local-volume-provider (LVP) plugin supports only Restic. Velero 1.17 and later do not support LVP. By default, KOTS uses the S3-compatible filesystem MinIO path for NFS storage when MinIO is enabled. KOTS uses LVP only when you disable MinIO or explicitly install the LVP plugin. For more information, see [Upgrade Velero for snapshots](/enterprise/snapshots-velero-upgrading). +::: + ### Usage ```bash diff --git a/docs/vendor/snapshots-overview.mdx b/docs/vendor/snapshots-overview.mdx index 8810a65377..90efd22ba8 100644 --- a/docs/vendor/snapshots-overview.mdx +++ b/docs/vendor/snapshots-overview.mdx @@ -10,7 +10,7 @@ import KotsAvailability from "../partials/kots/_kots-availability.mdx" # About backup and restore with snapshots -This topic provides an introduction to the Replicated KOTS snapshots feature for backup and restore. It describes how vendors enable snapshots, the type of data that is backed up, and how to troubleshoot issues for enterprise users. +This topic provides an introduction to the Replicated KOTS snapshots feature for backup and restore. It describes how vendors enable snapshots, the types of data that snapshots back up, and how to troubleshoot issues for enterprise users. :::note @@ -22,11 +22,11 @@ This topic provides an introduction to the Replicated KOTS snapshots feature for An important part of the lifecycle of an application is backup and restore. You can enable Replicated KOTS snapshots to support backup and restore for existing cluster installations with KOTS and Replicated kURL installations. -When snapshots is enabled for your application, your customers can manage and perform backup and restore from the Admin Console or KOTS CLI. +When you enable snapshots for your application, your customers can manage and perform backup and restore from the Admin Console or KOTS CLI. Snapshots uses the Velero open source project as the backend to back up Kubernetes manifests and persistent volumes. Velero is a mature, fully-featured application. For more information, see the [Velero documentation](https://velero.io/docs/). -In addition to the default functionality that Velero provides, KOTS exposes hooks that let you inject scripts that can execute both before and after a backup, and before and after a restore. For more information, see [Configure Backup and Restore Hooks for Snapshots](/vendor/snapshots-hooks). +In addition to the default functionality that Velero provides, KOTS exposes hooks that let you inject scripts. These scripts can execute both before and after a backup, and before and after a restore. For more information, see [Configure Backup and Restore Hooks for Snapshots](/vendor/snapshots-hooks). ### Limitations and considerations @@ -34,7 +34,7 @@ In addition to the default functionality that Velero provides, KOTS exposes hook - The snapshots feature is available only for licenses with the **Allow Snapshots** option enabled. For more information, see [Create and Manage Customers](/vendor/releases-creating-customer). -- Snapshots are useful for rollback and disaster recovery scenarios. They are not intended to be used for application migration. +- Snapshots are useful for rollback and disaster recovery scenarios. Do not use them for application migration. - @@ -54,31 +54,31 @@ In addition to the default functionality that Velero provides, KOTS exposes hook ## About backups -This section describes the types of backups that are supported with snapshots. For information about how to configure backup storage destinations for snapshots, see the [Configuring Backup Storage](/enterprise/snapshots-velero-cli-installing) section. +This section describes the backup types that the snapshots feature supports. For information about how to configure backup storage destinations for snapshots, see the [Configuring Backup Storage](/enterprise/snapshots-velero-cli-installing) section. ### Application and Admin Console (full) backups Full backups (also referred to as _instance_ backups) include the KOTS Admin Console and all application data, including application volumes and manifest files. -For clusters created with Replicated kURL, full backups also back up the Docker registry, which is required for air gapped installations. +For clusters created with Replicated kURL, full backups also back up the Docker registry. Air-gapped installations require this registry. -If you manage multiple applications with the Admin Console, data from all applications that support backups is included in a full backup. To be included in full backups, each application must include a manifest file with `kind: Backup` and `apiVersion: velero.io/v1`, which you can check for in the Admin Console. +If you manage multiple applications with the Admin Console, a full backup includes data from all applications that support backups. Each application must include a manifest file with `kind: Backup` and `apiVersion: velero.io/v1`. KOTS includes the application in full backups only when this manifest file is present. You can check for this manifest file in the Admin Console. -Full backups are recommended because they support all types of restores. For example, you can restore both the Admin Console and application from a full backup to a new cluster in disaster recovery scenarios. Or, you can use a full backup to restore only application data for the purpose of rolling back after deploying a new version of an application. +Replicated recommends full backups because they support all types of restores. For example, you can restore both the Admin Console and application from a full backup to a new cluster in disaster recovery scenarios. Or, you can use a full backup to restore only application data for the purpose of rolling back after deploying a new version of an application. ### Application-only (partial) backups Partial backups back up the application volumes and manifest files only. Partial backups do not back up the KOTS Admin Console. -Partial backups can be useful if you need to roll back after deploying a new application version. Partial backups of the application only _cannot_ be restored to a new cluster, and are therefore not useable for disaster recovery scenarios. +Partial backups can be useful if you need to roll back after deploying a new application version. You cannot restore partial backups of the application only to a new cluster, so they are not usable for disaster recovery scenarios. ### Backup storage destinations -For disaster recovery, backups should be configured to use a storage destination that exists outside of the cluster. This is especially true for installations in clusters created with Replicated kURL, because the default storage location on these clusters is internal. +For disaster recovery, configure backups to use a storage destination that exists outside of the cluster. This is especially true for installations in clusters created with Replicated kURL, because the default storage location on these clusters is internal. -You can use a storage provider that is compatible with Velero as the storage destination for backups created with the Replicated snapshots feature. For a list of the compatible storage providers, see [Providers](https://velero.io/docs/v1.9/supported-providers/) in the Velero documentation. +You can use a storage provider that is compatible with Velero as the storage destination for backups created with the Replicated snapshots feature. For a list of the compatible storage providers, see [Providers](https://velero.io/docs/v1.17/supported-providers/) in the Velero documentation. -You initially configure backups on a supported storage provider backend using the KOTS CLI. If you want to change the storage destination after the initial configuration, you can use the the **Snapshots** page in the Admin Console, which has built-in support for the following storage destinations: +You initially configure backups on a supported storage provider backend using the KOTS CLI. To change the storage destination after the initial configuration, use the **Snapshots** page in the Admin Console. The Admin Console supports the following storage destinations: - Amazon Web Services (AWS) - Google Cloud Provider (GCP) @@ -91,19 +91,19 @@ kURL installers that include the Velero add-on also include a locally-provisione For information about how to configure backup storage destinations for snapshots, see the [Configuring Backup Storage](/enterprise/snapshots-velero-cli-installing) section. -### What data is backed up? +### What data do backups include? -Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, this also backs up the Docker registry, which is required for air gapped installations. +Full backups include the Admin Console and all application data, including KOTS-specific object-stored data. For Replicated kURL installations, full backups also back up the Docker registry. Air-gapped installations require this registry. #### Other object-stored data For kURL clusters, you might be using object-stored data that is not specific to the kURL KOTS add-on. -For object-stored data that is not KOTS-specific and does not use persistentVolumeClaims (PVCs), you must write custom backup and restore hooks to enable back ups for that object-stored data. For example, Rook and Ceph do not use PVCs and so require custom backup and restore hooks. For more information about writing custom hooks, see [Configure Backup and Restore Hooks for Snapshots](snapshots-hooks). +For object-stored data that is not KOTS-specific and does not use persistentVolumeClaims (PVCs), you must write custom backup and restore hooks. For example, Rook, Ceph, and similar storage providers do not use PVCs and require custom backup and restore hooks. For more information about writing custom hooks, see [Configure Backup and Restore Hooks for Snapshots](snapshots-hooks). #### Pod volume data -Replicated supports only the restic backup program for pod volume data. +KOTS uses Velero's file-system backup uploader to back up pod volume data. For Velero 1.16 and earlier, the uploader is Restic. For Velero 1.17 and later, the uploader is Kopia. By default, Velero requires that you opt-in to have pod volumes backed up. In the Backup resource that you configure to enable snapshots, you must annotate each specific volume that you want to back up. For more information about including and excluding pod volumes, see [Configure Snapshots](/vendor/snapshots-configuring-backups). @@ -111,11 +111,11 @@ By default, Velero requires that you opt-in to have pod volumes backed up. In th -When you restore an application with snapshots, KOTS first deletes the selected application. All existing application manifests are removed from the cluster, and all `PersistentVolumeClaims` are deleted. This action is not reversible. +When you restore an application with snapshots, KOTS first deletes the selected application. KOTS removes all existing application manifests from the cluster and deletes all `PersistentVolumeClaims`. This action is not reversible. -Then, the restore process redeploys all of the application manifests. All Pods are given an extra `initContainer` and an extra directory named `.velero`, which are used for restore hooks. For more information about the restore process, see [Restore Reference](https://velero.io/docs/v1.9/restore-reference/) in the Velero documentation. +Then, the restore process redeploys all of the application manifests. The restore process gives all Pods an extra `initContainer` and an extra directory named `.velero`. KOTS uses these for restore hooks. For more information about the restore process, see [Restore Reference](https://velero.io/docs/v1.18/restore-reference/) in the Velero documentation. -When you restore the Admin Console only, no changes are made to the application. +When you restore the Admin Console only, the restore process makes no changes to the application. For information about how to restore using the Admin Console or the KOTS CLI, see [Restore from Backups](/enterprise/snapshots-restoring-full). @@ -127,7 +127,7 @@ This section provides an overview of how vendors and enterprise users can config To enable the snapshots backup and restore feature for your users, you must: -- Have the snapshots entitlement enabled in your Replicated vendor account. For account entitlements, contact the Replicated TAM team. +- Have the snapshots entitlement enabled in your Replicated vendor account. For account entitlements, contact the Replicated Technical Account Manager (TAM) team. - Define a manifest for creating backups. See [Configure Snapshots](snapshots-configuring-backups). - When needed, configure backup and restore hooks. See [Configure Backup and Restore Hooks for Snapshots](snapshots-hooks). - Enable the **Allow Snapshot** option in customer licenses. See [Create and Manage Customers](releases-creating-customer). @@ -142,7 +142,7 @@ From a full backup, users restore using the KOTS CLI or the Admin Console as ind -Partial backups are not recommended as they are a legacy feature and only back up the application volumes and manifests. Partial backups can be restored only from the Admin Console. +Partial backups are not recommended as they are a legacy feature and only back up the application volumes and manifests. Users can restore partial backups only from the Admin Console. ### Troubleshooting snapshots diff --git a/sidebars.js b/sidebars.js index f32c9d0225..4261529f7f 100644 --- a/sidebars.js +++ b/sidebars.js @@ -395,7 +395,7 @@ const sidebars = { }, { type: "category", - label: "Configure Backup Storage for Snaphots", + label: "Configure Backup Storage for Snapshots", items: [ "enterprise/snapshots-velero-cli-installing", "enterprise/snapshots-configuring-hostpath", @@ -408,6 +408,8 @@ const sidebars = { "enterprise/snapshots-restoring-full", "enterprise/snapshots-updating-with-admin-console", "enterprise/snapshots-troubleshooting-backup-restore", + "enterprise/snapshots-troubleshooting-reclaim-space", + "enterprise/snapshots-velero-upgrading", ], }, { diff --git a/styles/Replicated/Acronyms.yml b/styles/Replicated/Acronyms.yml index a9c1db766b..c15b2b2c44 100644 --- a/styles/Replicated/Acronyms.yml +++ b/styles/Replicated/Acronyms.yml @@ -57,6 +57,7 @@ exceptions: - SSH - SSL - SVG + - TAR - TBD - TCP - TODO @@ -64,6 +65,7 @@ exceptions: - URL - USB - UTF + - UID - UUID - VXLAN - XML diff --git a/styles/config/vocabularies/TechJargon/accept.txt b/styles/config/vocabularies/TechJargon/accept.txt index b8e9634d81..f9bfdd9783 100644 --- a/styles/config/vocabularies/TechJargon/accept.txt +++ b/styles/config/vocabularies/TechJargon/accept.txt @@ -94,6 +94,7 @@ rollout SAML SCIM SSO +TAR TOTP ttl keyless @@ -124,3 +125,9 @@ discoverability idempotently monorepo Monorepo +LVP +uploader +CRs +BackupRepository +preconfigured +crashloop diff --git a/styles/config/vocabularies/ThirdPartyProducts/accept.txt b/styles/config/vocabularies/ThirdPartyProducts/accept.txt index 755fe64a59..0f0846b13e 100644 --- a/styles/config/vocabularies/ThirdPartyProducts/accept.txt +++ b/styles/config/vocabularies/ThirdPartyProducts/accept.txt @@ -49,3 +49,10 @@ Fulcio Rekor SecureBuild Sigstore +Kopia +Restic +MinIO +Minio +Ceph +VMware +Tanzu