From bb87c7968c902d693b43d08f5e336c0c4a793c22 Mon Sep 17 00:00:00 2001 From: Tingmao Wang Date: Sun, 19 Jul 2026 22:35:50 +0000 Subject: [PATCH 1/4] securitypolicy_options: Write enhanced fragment loading diagnostics Write the error string or a success marker, the decoded fragment content, and headers, and place them in a findable place on Windows and Linux. Assisted-by: GitHub-Copilot copilot-review Signed-off-by: Tingmao Wang --- internal/guestpath/paths.go | 6 ++ pkg/securitypolicy/securitypolicy_options.go | 106 ++++++++++++++++--- 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/internal/guestpath/paths.go b/internal/guestpath/paths.go index d616efc646..757c5fb229 100644 --- a/internal/guestpath/paths.go +++ b/internal/guestpath/paths.go @@ -11,6 +11,12 @@ const ( // WCOWRootPrefixInUVM is the path inside UVM where WCOW container's root // file system will be mounted WCOWRootPrefixInUVM = `C:\c` + // LCOWFragmentsPath is the path inside the UVM where injected security + // policy fragments are stored. + LCOWFragmentsPath = "/tmp/fragments" + // WCOWFragmentsPath is the path inside the UVM where injected security + // policy fragments are stored. + WCOWFragmentsPath = `C:\InjectedFragments` // SandboxMountPrefix is mount prefix used in container spec to mark a // sandbox-mount SandboxMountPrefix = "sandbox://" diff --git a/pkg/securitypolicy/securitypolicy_options.go b/pkg/securitypolicy/securitypolicy_options.go index ef37d144eb..22903c8269 100644 --- a/pkg/securitypolicy/securitypolicy_options.go +++ b/pkg/securitypolicy/securitypolicy_options.go @@ -4,16 +4,19 @@ import ( "context" "crypto/sha256" "encoding/base64" + "encoding/hex" + "encoding/json" "fmt" "io" "math" "os" "path/filepath" "sync" - "time" + "sync/atomic" "github.com/Microsoft/cosesign1go/pkg/cosesign1" didx509resolver "github.com/Microsoft/didx509go/pkg/did-x509-resolver" + "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/ot" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" @@ -35,6 +38,18 @@ type SecurityOptions struct { logWriter io.Writer } +// Global counter for fragment injection requests, used to create unique +// error / success marker files even when the same fragment is injected +// multiple times. +var FragmentRequestId atomic.Uint64 + +func fragmentsPath() string { + if osType == "windows" { + return guestpath.WCOWFragmentsPath + } + return guestpath.LCOWFragmentsPath +} + func NewSecurityOptions(enforcer SecurityPolicyEnforcer, enforcerSet bool, uvmReferenceInfo string, uvmHashEnvelopeReferenceInfo string, logWriter io.Writer) *SecurityOptions { return &SecurityOptions{ PolicyEnforcer: enforcer, @@ -59,6 +74,14 @@ func (s *SecurityOptions) SetConfidentialOptions(ctx context.Context, enforcerTy return errors.New("security policy has already been set") } + // Pre-create this directory so that we can mount this dir into + // containers even if no fragments have been injected yet when the + // container starts. + if err := os.MkdirAll(fragmentsPath(), 0755); err != nil { + // This is not fatal, don't fail here. + log.G(ctx).WithError(err).Error("failed to create injected fragments directory") + } + hostData, err := NewSecurityPolicyDigest(encodedSecurityPolicy) if err != nil { return err @@ -140,6 +163,42 @@ func asInt64(v interface{}) (int64, error) { } } +func writeFileIfNotExists(filename string, data []byte, perm os.FileMode) error { + file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm) + if os.IsExist(err) { + return nil + } + if err != nil { + return err + } + + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + return file.Close() +} + +func writeFragmentMetadata(ctx context.Context, filename, issuer, feed string, headerSVN *int64) { + metadata := struct { + Issuer string `json:"issuer"` + Feed string `json:"feed"` + HeaderSVN *int64 `json:"headerSvn"` + }{ + Issuer: issuer, + Feed: feed, + HeaderSVN: headerSVN, + } + contents, err := json.Marshal(metadata) + if err != nil { + log.G(ctx).WithError(err).Warn("failed to marshal injected fragment metadata") + return + } + if err := writeFileIfNotExists(filename, contents, 0644); err != nil { + log.G(ctx).WithError(err).Warn("failed to write injected fragment metadata") + } +} + // Fragment extends current security policy with additional constraints // from the incoming fragment. Note that it is base64 encoded over the bridge/ // @@ -155,6 +214,7 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres defer span.End() defer func() { ot.SetSpanStatus(span, err) }() span.SetAttributes(attribute.String("fragment", fmt.Sprintf("%+v", fragment))) + currReqId := FragmentRequestId.Add(1) // An empty media type defaults to a Rego policy fragment, for backward // compatibility with older hosts that do not set the field. @@ -162,6 +222,32 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres if mediaType == "" { mediaType = mediaTypeFragment } + + raw, err := base64.StdEncoding.DecodeString(fragment.Fragment) + if err != nil { + return fmt.Errorf("failed to decode fragment: %w", err) + } + sha := sha256.Sum256(raw) + shaHex := hex.EncodeToString(sha[:]) + thisFragmentDir := filepath.Join(fragmentsPath(), shaHex) + defer func() { + markerName := fmt.Sprintf("%d.succeed", currReqId) + var markerContents []byte + if err != nil { + markerName = fmt.Sprintf("%d.fail", currReqId) + markerContents = []byte(err.Error()) + } + if markerErr := os.WriteFile(filepath.Join(thisFragmentDir, markerName), markerContents, 0644); markerErr != nil { + log.G(ctx).WithError(markerErr).Warnf("failed to write injected fragment outcome marker %s", markerName) + } + }() + + if err := os.MkdirAll(thisFragmentDir, 0755); err != nil { + return fmt.Errorf("failed to create injected fragment directory: %w", err) + } + if err := writeFileIfNotExists(filepath.Join(thisFragmentDir, "fragment.cose"), raw, 0644); err != nil { + return fmt.Errorf("failed to write fragment.cose: %w", err) + } switch mediaType { case mediaTypeFragment, mediaTypeTransparencyTrustList: default: @@ -172,24 +258,13 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres return fmt.Errorf("cannot inject fragment blob with unsupported media type %q", mediaType) } - raw, err := base64.StdEncoding.DecodeString(fragment.Fragment) - if err != nil { - return fmt.Errorf("failed to decode fragment: %w", err) - } - blob := []byte(fragment.Fragment) - // keep a copy of the fragment, so we can manually figure out what went wrong - // will be removed eventually. Give it a unique name to avoid any potential - // race conditions. - sha := sha256.New() - sha.Write(blob) - timestamp := time.Now() - fragmentPath := fmt.Sprintf("fragment-%x-%d.blob", sha.Sum(nil), timestamp.UnixMilli()) - _ = os.WriteFile(filepath.Join(os.TempDir(), fragmentPath), blob, 0644) - unpacked, err := cosesign1.UnpackAndValidateCOSE1CertChain(raw) if err != nil { return fmt.Errorf("InjectFragment failed COSE validation: %w", err) } + if err := writeFileIfNotExists(filepath.Join(thisFragmentDir, "fragment"), unpacked.Payload, 0644); err != nil { + return fmt.Errorf("failed to write injected fragment payload: %w", err) + } cwtClaimsRaw, hasCwtClaims := unpacked.Protected[cosesign1.COSE_Header_CWTClaims] var cwtClaims map[any]any @@ -241,6 +316,7 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres svnFromCwt = &svn } } + writeFragmentMetadata(ctx, filepath.Join(thisFragmentDir, "metadata.json"), issuer, feed, svnFromCwt) switch mediaType { case mediaTypeTransparencyTrustList: From fabf3eb266335d2a58ed4503fcc9f7688cfe9e9b Mon Sep 17 00:00:00 2001 From: Tingmao Wang Date: Sun, 19 Jul 2026 22:53:53 +0000 Subject: [PATCH 2/4] gcs-sidecar: Mount WCOWFragmentsPath into container's securityContext\fragments.debug [cherry-pick of 43dcfafab onto main] Assisted-by: GitHub-Copilot copilot-review Signed-off-by: Tingmao Wang --- internal/gcs-sidecar/handlers.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go index 1cf1ebff39..7afab9f6ee 100644 --- a/internal/gcs-sidecar/handlers.go +++ b/internal/gcs-sidecar/handlers.go @@ -19,8 +19,10 @@ import ( "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/fsformatter" "github.com/Microsoft/hcsshim/internal/gcs/prot" + "github.com/Microsoft/hcsshim/internal/guestpath" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" oci "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/ot" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" @@ -159,8 +161,10 @@ func (b *Bridge) createContainer(req *request) (err error) { } }() + var securityContextDir string + if oci.ParseAnnotationsBool(ctx, spec.Annotations, annotations.WCOWSecurityPolicyEnv, true) { - securityContextDir, err := b.hostState.securityOptions.WriteSecurityContextDir(&spec) + securityContextDir, err = b.hostState.securityOptions.WriteSecurityContextDir(&spec) if err != nil { return fmt.Errorf("failed to write security context dir: %w", err) } @@ -177,6 +181,21 @@ func (b *Bridge) createContainer(req *request) (err error) { cwcowHostedSystemConfig.Spec = spec } + // Add this fragments.debug mount after policy enforcement and + // reconcile checks so the policy does not have to explicitly + // allow it. + if securityContextDir != "" { + if err := os.MkdirAll(guestpath.WCOWFragmentsPath, 0755); err != nil { + log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.WCOWFragmentsPath).Warn("failed to create fragments debug mount path in uVM") + } else { + container.MappedDirectories = append(container.MappedDirectories, hcsschema.MappedDirectory{ + HostPath: guestpath.WCOWFragmentsPath, + ContainerPath: filepath.Join(`C:\`, filepath.Base(securityContextDir), "fragments.debug"), + ReadOnly: true, + }) + } + } + // Strip the spec field hostedSystemBytes, err := json.Marshal(cwcowHostedSystem) From 71a0f55620c9b7d0345c3dca518e947f7da69656 Mon Sep 17 00:00:00 2001 From: Tingmao Wang Date: Sun, 19 Jul 2026 23:46:54 +0000 Subject: [PATCH 3/4] hcsv2/uvm: Mount LCOWFragmentsPath into container's securityContext/fragments.debug Assisted-by: GitHub-Copilot copilot-review Signed-off-by: Tingmao Wang --- internal/guest/runtime/hcsv2/uvm.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 0609966a0a..08039fa499 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -739,9 +739,24 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM } if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWSecurityPolicyEnv, true) { - if _, err := h.securityOptions.WriteSecurityContextDir(settings.OCISpecification); err != nil { + securityContextDir, err := h.securityOptions.WriteSecurityContextDir(settings.OCISpecification) + if err != nil { return nil, fmt.Errorf("failed to write security context dir: %w", err) } + // Add this special fragments debug mount after policy enforcement + // so the policy does not have to explicitly allow it. + if securityContextDir != "" { + if err := os.MkdirAll(guestpath.LCOWFragmentsPath, 0755); err != nil { + log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.LCOWFragmentsPath).Warn("failed to create fragments debug mount path in uVM") + } else { + settings.OCISpecification.Mounts = append(settings.OCISpecification.Mounts, specs.Mount{ + Destination: path.Join("/", filepath.Base(securityContextDir), "fragments.debug"), + Type: "bind", + Source: guestpath.LCOWFragmentsPath, + Options: []string{"bind", "ro"}, + }) + } + } } // Create the BundlePath From a66042e7ec80f78d4f7aacbc87a131366dc5fddc Mon Sep 17 00:00:00 2001 From: Tingmao Wang Date: Wed, 22 Jul 2026 14:54:37 +0000 Subject: [PATCH 4/4] Rename x.fail to x.deny, rename fragments.debug to fragments.info, and add a README We can have things we expect to "fail" e.g. old infra fragments with old svns that are always injected by the host for compatibility / rolling upgrade reasons. As such we rename x.fail to x.deny (even if it's not actually a deny but a genuine parse error etc) Assisted-by: GitHub-Copilot copilot-review Signed-off-by: Tingmao Wang --- internal/gcs-sidecar/handlers.go | 8 +++---- internal/guest/runtime/hcsv2/uvm.go | 8 +++---- pkg/securitypolicy/fragments_info_README | 1 + pkg/securitypolicy/securitypolicy_options.go | 22 ++++++++++++++++---- 4 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 pkg/securitypolicy/fragments_info_README diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go index 7afab9f6ee..dc0c9d93d5 100644 --- a/internal/gcs-sidecar/handlers.go +++ b/internal/gcs-sidecar/handlers.go @@ -181,16 +181,16 @@ func (b *Bridge) createContainer(req *request) (err error) { cwcowHostedSystemConfig.Spec = spec } - // Add this fragments.debug mount after policy enforcement and + // Add this fragments.info mount after policy enforcement and // reconcile checks so the policy does not have to explicitly // allow it. if securityContextDir != "" { - if err := os.MkdirAll(guestpath.WCOWFragmentsPath, 0755); err != nil { - log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.WCOWFragmentsPath).Warn("failed to create fragments debug mount path in uVM") + if err := b.hostState.securityOptions.EnsureFragmentDiagnosticsDir(); err != nil { + log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.WCOWFragmentsPath).Warn("failed to prepare fragments.info mount path in uVM") } else { container.MappedDirectories = append(container.MappedDirectories, hcsschema.MappedDirectory{ HostPath: guestpath.WCOWFragmentsPath, - ContainerPath: filepath.Join(`C:\`, filepath.Base(securityContextDir), "fragments.debug"), + ContainerPath: filepath.Join(`C:\`, filepath.Base(securityContextDir), "fragments.info"), ReadOnly: true, }) } diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 08039fa499..314d4f0216 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -743,14 +743,14 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM if err != nil { return nil, fmt.Errorf("failed to write security context dir: %w", err) } - // Add this special fragments debug mount after policy enforcement + // Add this special fragments.info mount after policy enforcement // so the policy does not have to explicitly allow it. if securityContextDir != "" { - if err := os.MkdirAll(guestpath.LCOWFragmentsPath, 0755); err != nil { - log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.LCOWFragmentsPath).Warn("failed to create fragments debug mount path in uVM") + if err := h.securityOptions.EnsureFragmentDiagnosticsDir(); err != nil { + log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.LCOWFragmentsPath).Warn("failed to prepare fragments.info mount path in uVM") } else { settings.OCISpecification.Mounts = append(settings.OCISpecification.Mounts, specs.Mount{ - Destination: path.Join("/", filepath.Base(securityContextDir), "fragments.debug"), + Destination: path.Join("/", filepath.Base(securityContextDir), "fragments.info"), Type: "bind", Source: guestpath.LCOWFragmentsPath, Options: []string{"bind", "ro"}, diff --git a/pkg/securitypolicy/fragments_info_README b/pkg/securitypolicy/fragments_info_README new file mode 100644 index 0000000000..f928c208ae --- /dev/null +++ b/pkg/securitypolicy/fragments_info_README @@ -0,0 +1 @@ +The content of this directory is for informational purpose and should not be relied upon for security. diff --git a/pkg/securitypolicy/securitypolicy_options.go b/pkg/securitypolicy/securitypolicy_options.go index 22903c8269..00adb573fa 100644 --- a/pkg/securitypolicy/securitypolicy_options.go +++ b/pkg/securitypolicy/securitypolicy_options.go @@ -3,6 +3,7 @@ package securitypolicy import ( "context" "crypto/sha256" + _ "embed" "encoding/base64" "encoding/hex" "encoding/json" @@ -39,10 +40,13 @@ type SecurityOptions struct { } // Global counter for fragment injection requests, used to create unique -// error / success marker files even when the same fragment is injected +// deny / success marker files even when the same fragment is injected // multiple times. var FragmentRequestId atomic.Uint64 +//go:embed fragments_info_README +var fragmentsInfoREADME []byte + func fragmentsPath() string { if osType == "windows" { return guestpath.WCOWFragmentsPath @@ -50,6 +54,16 @@ func fragmentsPath() string { return guestpath.LCOWFragmentsPath } +// EnsureFragmentDiagnosticsDir creates the fragment diagnostics directory and +// its informational README. +func (s *SecurityOptions) EnsureFragmentDiagnosticsDir() error { + dir := fragmentsPath() + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "README"), fragmentsInfoREADME, 0644) +} + func NewSecurityOptions(enforcer SecurityPolicyEnforcer, enforcerSet bool, uvmReferenceInfo string, uvmHashEnvelopeReferenceInfo string, logWriter io.Writer) *SecurityOptions { return &SecurityOptions{ PolicyEnforcer: enforcer, @@ -77,9 +91,9 @@ func (s *SecurityOptions) SetConfidentialOptions(ctx context.Context, enforcerTy // Pre-create this directory so that we can mount this dir into // containers even if no fragments have been injected yet when the // container starts. - if err := os.MkdirAll(fragmentsPath(), 0755); err != nil { + if err := s.EnsureFragmentDiagnosticsDir(); err != nil { // This is not fatal, don't fail here. - log.G(ctx).WithError(err).Error("failed to create injected fragments directory") + log.G(ctx).WithError(err).Error("failed to prepare injected fragments directory") } hostData, err := NewSecurityPolicyDigest(encodedSecurityPolicy) @@ -234,7 +248,7 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres markerName := fmt.Sprintf("%d.succeed", currReqId) var markerContents []byte if err != nil { - markerName = fmt.Sprintf("%d.fail", currReqId) + markerName = fmt.Sprintf("%d.deny", currReqId) markerContents = []byte(err.Error()) } if markerErr := os.WriteFile(filepath.Join(thisFragmentDir, markerName), markerContents, 0644); markerErr != nil {