Skip to content

Commit 500e328

Browse files
committed
refactor drivers util methods
In order to install device drivers, the following workflow is followed- - Make the device driver available within the UVM via SCSI or VSMB. - Use `install-drivers` for LCOW and `pnputils` for WCOW to install the driver Presently, everything was clustered together in `internal/devices` package. The utility methods to perform step 2 above are not tied to devices per se and belong to the `drivers`. Since we can easily re-use these methods, we are refactoring them into `internal/controller/drivers` where they exist as utility methods and are used in both new and old shims. Signed-off-by: Harsh Rawat <harshrawat@microsoft.com>
1 parent 87708ff commit 500e328

8 files changed

Lines changed: 251 additions & 196 deletions

File tree

internal/controller/drivers/doc.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
//go:build windows
2+
3+
// Package drivers provides utility methods for installing drivers into
4+
// Linux or Windows utility VMs (UVMs).
5+
//
6+
// These utility methods are used by 'containerd-shim-runhcs-v1' as well as
7+
// V2 shims to install the drivers.
8+
//
9+
// For LCOW guests, driver installation is performed by executing the
10+
// 'install-drivers' binary inside the guest via [ExecGCSInstallDriver].
11+
//
12+
// For WCOW guests, driver installation is performed by invoking 'pnputil'
13+
// inside the UVM via [ExecPnPInstallDriver].
14+
package drivers
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
//go:build windows
2+
3+
package drivers
4+
5+
import (
6+
"context"
7+
"errors"
8+
"fmt"
9+
"io"
10+
"net"
11+
12+
"github.com/Microsoft/go-winio/pkg/guid"
13+
14+
"github.com/Microsoft/hcsshim/internal/cmd"
15+
"github.com/Microsoft/hcsshim/internal/guestpath"
16+
)
17+
18+
var noExecOutputErr = errors.New("failed to get any pipe output")
19+
20+
// guest is the UVM instance in which the driver will be installed.
21+
type guest interface {
22+
ExecInUVM(ctx context.Context, request *cmd.CmdProcessRequest) (int, error)
23+
}
24+
25+
// ExecGCSInstallDriver installs a driver into the UVM by running 'install-drivers'
26+
// inside the guest. hostPath is the host VHD path and guestPath is the
27+
// SCSI-mounted location inside the UVM. Returns an error if installation fails,
28+
// along with any stderr output from the guest process.
29+
func ExecGCSInstallDriver(ctx context.Context, guest guest, hostPath string, guestPath string) error {
30+
driverReadWriteDir, err := getDriverWorkDir(hostPath)
31+
if err != nil {
32+
return fmt.Errorf("failed to create a guid path for driver %+v: %w", hostPath, err)
33+
}
34+
35+
p, l, err := cmd.CreateNamedPipeListener()
36+
if err != nil {
37+
return err
38+
}
39+
defer l.Close()
40+
41+
var stderrOutput string
42+
errChan := make(chan error)
43+
44+
go readAllPipeOutput(l, errChan, &stderrOutput)
45+
46+
args := []string{
47+
"/bin/install-drivers",
48+
driverReadWriteDir,
49+
guestPath,
50+
}
51+
req := &cmd.CmdProcessRequest{
52+
Args: args,
53+
Stderr: p,
54+
}
55+
56+
// A call to `ExecInUvm` may fail in the following ways:
57+
// - The process runs and exits with a non-zero exit code. In this case we need to wait on the output
58+
// from stderr so we can log it for debugging.
59+
// - There's an error trying to run the process. No need to wait for stderr logs.
60+
// - There's an error copying IO. No need to wait for stderr logs.
61+
//
62+
// Since we cannot distinguish between the cases above, we should always wait to read the stderr output.
63+
exitCode, execErr := guest.ExecInUVM(ctx, req)
64+
65+
// wait to finish parsing stdout results
66+
select {
67+
case err := <-errChan:
68+
if err != nil && !errors.Is(err, noExecOutputErr) {
69+
return fmt.Errorf("failed to get stderr output from command %s: %w", guestPath, err)
70+
}
71+
case <-ctx.Done():
72+
return fmt.Errorf("timed out waiting for the console output from installing driver %s: %w", guestPath, ctx.Err())
73+
}
74+
75+
if execErr != nil {
76+
return fmt.Errorf("%w: failed to install driver %s in uvm with exit code %d: %v", execErr, guestPath, exitCode, stderrOutput)
77+
}
78+
return nil
79+
}
80+
81+
// getDriverWorkDir returns the deterministic guest path used as the overlayfs
82+
// root for a driver installation. 'install-drivers' uses the read-only SCSI VHD
83+
// as the lower layer and uses this directory for the upper, work, and content
84+
// directories, giving depmod/modprobe a writable view.
85+
//
86+
// If the directory already exists, 'install-drivers' skips reinstallation.
87+
// The path is derived from a v5 UUID seeded with the host VHD path,
88+
// ensuring a stable mapping across reboots.
89+
func getDriverWorkDir(hostPath string) (string, error) {
90+
// 914aadc8-f700-4365-8016-ddad0a9d406d. Random GUID chosen for namespace.
91+
ns := guid.GUID{
92+
Data1: 0x914aadc8,
93+
Data2: 0xf700,
94+
Data3: 0x4365,
95+
Data4: [8]byte{0x80, 0x16, 0xdd, 0xad, 0x0a, 0x9d, 0x40, 0x6d},
96+
}
97+
98+
driverGUID, err := guid.NewV5(ns, []byte(hostPath))
99+
if err != nil {
100+
return "", err
101+
}
102+
103+
return fmt.Sprintf(guestpath.LCOWGlobalDriverPrefixFmt, driverGUID.String()), nil
104+
}
105+
106+
// readAllPipeOutput is a helper function that connects to a listener and attempts to
107+
// read the connection's entire output. Resulting output is returned as a string
108+
// in the `result` param. The `errChan` param is used to propagate an errors to
109+
// the calling function.
110+
func readAllPipeOutput(l net.Listener, errChan chan<- error, result *string) {
111+
defer close(errChan)
112+
c, err := l.Accept()
113+
if err != nil {
114+
errChan <- fmt.Errorf("failed to accept named pipe: %w", err)
115+
return
116+
}
117+
bytes, err := io.ReadAll(c)
118+
if err != nil {
119+
errChan <- err
120+
return
121+
}
122+
123+
*result = string(bytes)
124+
125+
if len(*result) == 0 {
126+
errChan <- noExecOutputErr
127+
return
128+
}
129+
130+
errChan <- nil
131+
}
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
//go:build windows
2+
3+
package drivers
4+
5+
import (
6+
"context"
7+
"fmt"
8+
9+
"github.com/Microsoft/hcsshim/internal/cmd"
10+
"github.com/Microsoft/hcsshim/internal/log"
11+
"github.com/Microsoft/hcsshim/internal/logfields"
12+
"github.com/Microsoft/hcsshim/internal/uvm"
13+
"github.com/Microsoft/hcsshim/internal/winapi"
14+
15+
"github.com/sirupsen/logrus"
16+
)
17+
18+
const (
19+
uvmPnpExePath = "C:\\Windows\\System32\\pnputil.exe"
20+
pnputilNoMoreItemsErrorMessage = `driver not ranked higher than existing driver in UVM.
21+
if drivers were not previously present in the UVM, this
22+
is an expected race and can be ignored.`
23+
)
24+
25+
// createPnPInstallDriverCommand creates a pnputil command to add and install drivers
26+
// present in `driverUVMPath` and all subdirectories.
27+
func createPnPInstallDriverCommand(driverUVMPath string) []string {
28+
dirFormatted := fmt.Sprintf("%s/*.inf", driverUVMPath)
29+
args := []string{
30+
"cmd",
31+
"/c",
32+
uvmPnpExePath,
33+
"/add-driver",
34+
dirFormatted,
35+
"/subdirs",
36+
"/install",
37+
}
38+
return args
39+
}
40+
41+
// ExecPnPInstallDriver makes the calls to exec in the uvm the pnp command
42+
// that installs a driver previously mounted into the uvm.
43+
func ExecPnPInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir string) error {
44+
args := createPnPInstallDriverCommand(driverDir)
45+
cmdReq := &cmd.CmdProcessRequest{
46+
Args: args,
47+
}
48+
exitCode, err := vm.ExecInUVM(ctx, cmdReq)
49+
if err != nil && exitCode != winapi.ERROR_NO_MORE_ITEMS {
50+
return fmt.Errorf("failed to install driver %s in uvm with exit code %d: %w", driverDir, exitCode, err)
51+
} else if exitCode == winapi.ERROR_NO_MORE_ITEMS {
52+
// As mentioned in `pnputilNoMoreItemsErrorMessage`, this exit code comes from pnputil
53+
// but is not necessarily an error
54+
log.G(ctx).WithFields(logrus.Fields{
55+
logfields.UVMID: vm.ID(),
56+
"driver": driverDir,
57+
"error": pnputilNoMoreItemsErrorMessage,
58+
}).Warn("expected version of driver may not have been installed")
59+
}
60+
61+
log.G(ctx).WithField("added drivers", driverDir).Debug("installed drivers")
62+
return nil
63+
}

internal/controller/vm/vm.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (c *Manager) ExecIntoHost(ctx context.Context, request *shimdiag.ExecProces
252252
Stdout: request.Stdout,
253253
Stderr: request.Stderr,
254254
}
255-
return c.guest.ExecIntoUVM(ctx, cmdReq)
255+
return c.guest.ExecInUVM(ctx, cmdReq)
256256
}
257257

258258
// DumpStacks dumps the GCS stacks associated with the VM

internal/devices/assigned_devices.go

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@ package devices
55

66
import (
77
"context"
8+
"errors"
89
"fmt"
10+
"io"
11+
"net"
912
"path/filepath"
1013
"strconv"
14+
"strings"
1115

1216
"github.com/Microsoft/hcsshim/internal/cmd"
1317
"github.com/Microsoft/hcsshim/internal/log"
1418
"github.com/Microsoft/hcsshim/internal/uvm"
15-
"github.com/pkg/errors"
1619
)
1720

1821
// AddDevice is the api exposed to oci/hcsoci to handle assigning a device on a WCOW UVM
@@ -43,7 +46,7 @@ func AddDevice(ctx context.Context, vm *uvm.UtilityVM, idType, deviceID string,
4346
if uvm.IsValidDeviceType(idType) {
4447
vpci, err = vm.AssignDevice(ctx, deviceID, index, "")
4548
if err != nil {
46-
return vpci, nil, errors.Wrapf(err, "failed to assign device %s of type %s to pod %s", deviceID, idType, vm.ID())
49+
return vpci, nil, fmt.Errorf("failed to assign device %s of type %s to pod %s: %w", deviceID, idType, vm.ID(), err)
4750
}
4851
vmBusInstanceID := vm.GetAssignedDeviceVMBUSInstanceID(vpci.VMBusGUID)
4952
log.G(ctx).WithField("vmbus id", vmBusInstanceID).Info("vmbus instance ID")
@@ -77,7 +80,7 @@ func getChildrenDeviceLocationPaths(ctx context.Context, vm *uvm.UtilityVM, vmBu
7780
}
7881
exitCode, err := vm.ExecInUVM(ctx, cmdReq)
7982
if err != nil {
80-
return nil, errors.Wrapf(err, "failed to find devices with exit code %d", exitCode)
83+
return nil, fmt.Errorf("failed to find devices with exit code %d: %w", exitCode, err)
8184
}
8285

8386
// wait to finish parsing stdout results
@@ -120,3 +123,32 @@ func GetDeviceInfoFromPath(rawDevicePath string) (string, uint16) {
120123
// otherwise, just use default index and full device ID given
121124
return rawDevicePath, 0
122125
}
126+
127+
// readCsPipeOutput is a helper function that connects to a listener and reads
128+
// the connection's comma separated output until done. resulting comma separated
129+
// values are returned in the `result` param. The `errChan` param is used to
130+
// propagate an errors to the calling function.
131+
func readCsPipeOutput(l net.Listener, errChan chan<- error, result *[]string) {
132+
defer close(errChan)
133+
c, err := l.Accept()
134+
if err != nil {
135+
errChan <- fmt.Errorf("failed to accept named pipe: %w", err)
136+
return
137+
}
138+
bytes, err := io.ReadAll(c)
139+
if err != nil {
140+
errChan <- err
141+
return
142+
}
143+
144+
elementsAsString := strings.TrimSuffix(string(bytes), "\n")
145+
elements := strings.Split(elementsAsString, ",")
146+
*result = append(*result, elements...)
147+
148+
if len(*result) == 0 {
149+
errChan <- errors.New("failed to get any pipe output")
150+
return
151+
}
152+
153+
errChan <- nil
154+
}

internal/devices/drivers.go

Lines changed: 3 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,9 @@ package devices
55

66
import (
77
"context"
8-
"errors"
98
"fmt"
109

11-
"github.com/Microsoft/go-winio/pkg/guid"
12-
"github.com/Microsoft/hcsshim/internal/cmd"
13-
"github.com/Microsoft/hcsshim/internal/guestpath"
10+
"github.com/Microsoft/hcsshim/internal/controller/drivers"
1411
"github.com/Microsoft/hcsshim/internal/log"
1512
"github.com/Microsoft/hcsshim/internal/resources"
1613
"github.com/Microsoft/hcsshim/internal/uvm"
@@ -51,7 +48,7 @@ func InstallDrivers(ctx context.Context, vm *uvm.UtilityVM, share string) (close
5148
}
5249
// attempt to install even if the driver has already been installed before so we
5350
// can guarantee the device is ready for use afterwards
54-
return closer, execPnPInstallDriver(ctx, vm, uvmPath)
51+
return closer, drivers.ExecPnPInstallDriver(ctx, vm, uvmPath)
5552
}
5653

5754
// first mount driver as scsi in standard mount location
@@ -69,63 +66,6 @@ func InstallDrivers(ctx context.Context, vm *uvm.UtilityVM, share string) (close
6966
closer = mount
7067
uvmPathForShare := mount.GuestPath()
7168

72-
// construct path that the drivers will be remounted as read/write in the UVM
73-
74-
// 914aadc8-f700-4365-8016-ddad0a9d406d. Random GUID chosen for namespace.
75-
ns := guid.GUID{Data1: 0x914aadc8, Data2: 0xf700, Data3: 0x4365, Data4: [8]byte{0x80, 0x16, 0xdd, 0xad, 0x0a, 0x9d, 0x40, 0x6d}}
76-
driverGUID, err := guid.NewV5(ns, []byte(share))
77-
if err != nil {
78-
return closer, fmt.Errorf("failed to create a guid path for driver %+v: %w", share, err)
79-
}
80-
uvmReadWritePath := fmt.Sprintf(guestpath.LCOWGlobalDriverPrefixFmt, driverGUID.String())
81-
8269
// install drivers using gcs tool `install-drivers`
83-
return closer, execGCSInstallDriver(ctx, vm, uvmPathForShare, uvmReadWritePath)
84-
}
85-
86-
func execGCSInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir string, driverReadWriteDir string) error {
87-
p, l, err := cmd.CreateNamedPipeListener()
88-
if err != nil {
89-
return err
90-
}
91-
defer l.Close()
92-
93-
var stderrOutput string
94-
errChan := make(chan error)
95-
96-
go readAllPipeOutput(l, errChan, &stderrOutput)
97-
98-
args := []string{
99-
"/bin/install-drivers",
100-
driverReadWriteDir,
101-
driverDir,
102-
}
103-
req := &cmd.CmdProcessRequest{
104-
Args: args,
105-
Stderr: p,
106-
}
107-
108-
// A call to `ExecInUvm` may fail in the following ways:
109-
// - The process runs and exits with a non-zero exit code. In this case we need to wait on the output
110-
// from stderr so we can log it for debugging.
111-
// - There's an error trying to run the process. No need to wait for stderr logs.
112-
// - There's an error copying IO. No need to wait for stderr logs.
113-
//
114-
// Since we cannot distinguish between the cases above, we should always wait to read the stderr output.
115-
exitCode, execErr := vm.ExecInUVM(ctx, req)
116-
117-
// wait to finish parsing stdout results
118-
select {
119-
case err := <-errChan:
120-
if err != nil && !errors.Is(err, noExecOutputErr) {
121-
return fmt.Errorf("failed to get stderr output from command %s: %w", driverDir, err)
122-
}
123-
case <-ctx.Done():
124-
return fmt.Errorf("timed out waiting for the console output from installing driver %s: %w", driverDir, ctx.Err())
125-
}
126-
127-
if execErr != nil {
128-
return fmt.Errorf("%w: failed to install driver %s in uvm with exit code %d: %v", execErr, driverDir, exitCode, stderrOutput)
129-
}
130-
return nil
70+
return closer, drivers.ExecGCSInstallDriver(ctx, vm, share, uvmPathForShare)
13171
}

0 commit comments

Comments
 (0)