Skip to content

feat: add erofs snapshotter configuration for new kata-preview runtime class #9151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Cameron E Baird (Camelron) wants to merge 4 commits into
base: main
Choose a base branch
from
Open

feat: add erofs snapshotter configuration for new kata-preview runtime class #9151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ab875bc
containerd: Configure erofs snapshotter for kata images
Camelron Jul 10, 2026
46bf53d
containerd: add erofs kata blocks to the aks-node-controller path
Camelron Jul 30, 2026
a1c762b
temp: Install erofs-utils for kata images
Camelron Jul 14, 2026
4ca38de
test(e2e): test kata-preview and erofs configurations for kata
Camelron Aug 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions aks-node-controller/parser/templates/containerd.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,27 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
enable_cdi = true
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if .GetIsKata }}
disable_snapshot_annotations = false
snapshotter = "overlayfs"
{{- end}}
Comment on lines 22 to 25
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -62,6 +77,7 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
Expand All @@ -74,6 +90,12 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-preview.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
22 changes: 22 additions & 0 deletions aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,26 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
[plugins."io.containerd.grpc.v1.cri".containerd]
{{- if .GetIsKata }}
disable_snapshot_annotations = false
snapshotter = "overlayfs"
{{- end}}
Comment thread
Camelron marked this conversation as resolved.
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -46,6 +61,7 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli]
runtime_type = "io.containerd.runc.v1"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.katacli.options]
Expand All @@ -58,6 +74,12 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
Root = ""
CriuPath = ""
SystemdCgroup = false
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-preview.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
21 changes: 21 additions & 0 deletions aks-node-controller/parser/templates/containerd_v2.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
Comment thread
Camelron marked this conversation as resolved.
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -56,8 +70,15 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
Comment thread
Camelron marked this conversation as resolved.
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-preview.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
21 changes: 21 additions & 0 deletions aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
version = 2
oom_score = -999{{if getHasDataDir .KubeletConfig}}
root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
{{- if .GetIsKata }}
[plugins."io.containerd.snapshotter.v1.erofs"]
default_size = "10G"
enable_fsverity = false
ovl_mount_options = []
max_unmerged_layers = 1

[plugins."io.containerd.service.v1.diff-service"]
default = ["erofs", "walking"]

[plugins."io.containerd.differ.v1.erofs"]
mkfs_options = ["-T0", "--mkfs-time", "--sort=none"]
enable_tar_index = false
{{- end}}
[plugins."io.containerd.cri.v1.images"]
{{- if .GetEnableArtifactStreaming }}
snapshotter = "overlaybd"
Expand Down Expand Up @@ -43,8 +57,15 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration.toml"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview]
runtime_type = "io.containerd.kata.v2"
privileged_without_host_devices = true
snapshotter = "erofs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-preview.options]
ConfigPath = "/usr/share/defaults/kata-containers/configuration-clh-preview.toml"
[proxy_plugins]
[proxy_plugins.tardev]
type = "snapshot"
Expand Down
3 changes: 2 additions & 1 deletion e2e/scenario_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@ func Test_AzureLinuxV3(t *testing.T) {
// scenario exercises that whole path against a real node.
//
// The scenario asserts three increasingly strong properties:
// 1. the rendered /etc/containerd/config.toml contains the Kata runtime handlers,
// 1. the rendered /etc/containerd/config.toml contains the Kata runtime handlers and EROFS preamble,
// 2. containerd actually parsed and loaded them (no warnings, handlers in `config dump`),
// 3. for every handler in kataRuntimeHandlers, a pod scheduled via a Kata RuntimeClass runs
// and is genuinely VM-isolated.
Expand All @@ -440,6 +440,7 @@ func Test_AzureLinuxV3Gen2Kata(t *testing.T) {
},
Validator: func(ctx context.Context, s *Scenario) {
ValidateKataContainerdConfig(ctx, s)
ValidateKataErofsContainerdConfig(ctx, s)
ValidateKataContainerdConfigDump(ctx, s)
ValidateKataHostReadiness(ctx, s)
for _, handler := range kataRuntimeHandlers {
Expand Down
27 changes: 24 additions & 3 deletions e2e/validators_kata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ import (
const (
// kataRuntimeHandler is the containerd runtime handler name for standard Kata Containers,
// emitted by the IsKata block of the containerd config templates in pkg/agent/baker.go.
kataRuntimeHandler = "kata"
kataRuntimeHandler = "kata"
kataPreviewRuntimeHandler = "kata-preview"

// kataConfigPath is the Kata configuration file referenced by the "kata" runtime handler's
// options.ConfigPath in the rendered containerd config.
Expand All @@ -30,12 +31,12 @@ const (
// kataRuntimeHandlers lists every Kata containerd runtime handler that this scenario expects to
// be configured and usable on the node. Each one is asserted in the effective containerd config
// and independently exercised by ValidateKataPodIsIsolated, so covering an additional handler is
// a one-line change here plus a call in the scenario.
// a one-line change here.
//
// Note that "kata-cc" (confidential containers) is intentionally absent: its handler block is
// templated for all Kata VHDs, but it targets a different VHD than regular Kata, so this image
// cannot actually run it.
var kataRuntimeHandlers = []string{kataRuntimeHandler}
var kataRuntimeHandlers = []string{kataRuntimeHandler, kataPreviewRuntimeHandler}

// ValidateKataContainerdConfig asserts that AgentBaker rendered a containerd configuration
// containing the Kata runtime handlers on a Kata-enabled VHD.
Expand Down Expand Up @@ -63,6 +64,26 @@ func ValidateKataContainerdConfig(ctx context.Context, s *Scenario) {
ValidateFileHasContent(ctx, s, containerdConfigPath, "disable_snapshot_annotations = false")
}

// ValidateKataErofsContainerdConfig checks that the EROFS snapshotter is configured and that
// containerd loaded all of its EROFS plugins successfully.
func ValidateKataErofsContainerdConfig(ctx context.Context, s *Scenario) {
s.T.Helper()

ValidateFileHasContent(ctx, s, containerdConfigPath, `[plugins."io.containerd.snapshotter.v1.erofs"]`)

execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s,
"sudo ctr plugins list | grep erofs", 0, "unable to list EROFS containerd plugins")
normalizedPluginList := strings.Join(strings.Fields(execResult.stdout), " ")
for _, expectedPlugin := range []string{
"io.containerd.mount-handler.v1 erofs linux/amd64 ok",
"io.containerd.snapshotter.v1 erofs linux/amd64 ok",
"io.containerd.differ.v1 erofs linux/amd64 ok",
} {
assert.Contains(s.T, normalizedPluginList, expectedPlugin,
"expected healthy EROFS plugin %q.\nPlugin list:\n%s", expectedPlugin, execResult.stdout)
}
}

// ValidateKataContainerdConfigDump asserts that containerd itself accepted the rendered
// configuration and actually loaded the Kata runtime handlers.
//
Expand Down
2 changes: 1 addition & 1 deletion parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ installDeps() {

installKataDeps() {
if [ "$OS_VERSION" != "1.0" ]; then
if ! dnf_install 30 1 600 kata-packages-host; then
if ! dnf_install 30 1 600 kata-packages-host erofs-utils; then

@Bickor Martin Heberling (Bickor) Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

3 questions:

  1. Will this still be required after our build uvm pipeline picks up the latest bits from your kata packages?
  2. Will there be any conflicts after this package is picked up by our build uvm pipeline?
  3. Do we need to run any changes in our build uvm pipeline once this is available? Or will it be automatically handled by the install packages?

@Camelron Cameron E Baird (Camelron) Aug 10, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The build uvm steps involve installing kata-packages-uvm into a guest disk. https://github.com/microsoft/azurelinux/blob/3.0-dev/SPECS/kata-packages-uvm/kata-packages-uvm.spec#L11

This change just adds an extra package erofs-utils to the host, on top of kata-package-host. The uvm image needs no separate change.

@Camelron Cameron E Baird (Camelron) Aug 10, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So in short, these are separate package lists that don't concern each other. Here we are installing an extra component on the host (and the next Azurelinux release will do this automatically). The guest components are separate and no change is needed there.

exit $ERR_APT_INSTALL_TIMEOUT
fi
fi
Expand Down
Loading
Loading