Merge pull request microsoft#1195 from katiewasnothere/lcow_install_m…

…odules

Add tool to install modules in lcow and plumb through shim

Makefile

@@ -18,7 +18,8 @@ SRCROOT=$(dir $(abspath $(firstword $(MAKEFILE_LIST))))
 
 # The link aliases for gcstools
 GCS_TOOLS=\
-	generichook
+	generichook \
+	install-drivers
 
 .PHONY: all always rootfs test
 

cmd/gcstools/generichook.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package main
 
 import (

cmd/gcstools/installdrivers.go

@@ -0,0 +1,96 @@
+// +build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/Microsoft/hcsshim/internal/guest/storage/overlay"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	lcowGlobalDriversFormat = "/run/drivers/%s"
+
+	moduleExtension = ".ko"
+)
+
+func install(ctx context.Context) error {
+	args := []string(os.Args[1:])
+
+	if len(args) == 0 {
+		return errors.New("no driver paths provided for install")
+	}
+
+	for _, driver := range args {
+		modules := []string{}
+
+		driverGUID, err := uuid.NewRandom()
+		if err != nil {
+			return err
+		}
+
+		// create an overlay mount from the driver's UVM path so we can write to the
+		// mount path in the UVM despite having mounted in the driver originally as
+		// readonly
+		runDriverPath := fmt.Sprintf(lcowGlobalDriversFormat, driverGUID.String())
+		upperPath := filepath.Join(runDriverPath, "upper")
+		workPath := filepath.Join(runDriverPath, "work")
+		rootPath := filepath.Join(runDriverPath, "content")
+		if err := overlay.Mount(ctx, []string{driver}, upperPath, workPath, rootPath, false); err != nil {
+			return err
+		}
+
+		// find all module files, which end with ".ko" extension, and remove extension
+		// for use when calling `modprobe` below.
+		if walkErr := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return errors.Wrap(err, "failed to read directory while walking dir")
+			}
+			if !info.IsDir() && filepath.Ext(info.Name()) == moduleExtension {
+				moduleName := strings.TrimSuffix(info.Name(), moduleExtension)
+				modules = append(modules, moduleName)
+			}
+			return nil
+		}); walkErr != nil {
+			return walkErr
+		}
+
+		// create a new module dependency map database for the driver
+		depmodArgs := []string{"-b", rootPath}
+		cmd := exec.Command("depmod", depmodArgs...)
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrapf(err, "failed to run depmod with args %v: %s", depmodArgs, out)
+		}
+
+		// run modprobe for every module name found
+		modprobeArgs := append([]string{"-d", rootPath, "-a"}, modules...)
+		cmd = exec.Command(
+			"modprobe",
+			modprobeArgs...,
+		)
+
+		out, err = cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrapf(err, "failed to run modporbe with args %v: %s", modprobeArgs, out)
+		}
+	}
+
+	return nil
+}
+
+func installDriversMain() {
+	ctx := context.Background()
+	logrus.SetOutput(os.Stderr)
+	if err := install(ctx); err != nil {
+		logrus.Fatalf("error in install drivers: %s", err)
+	}
+}

cmd/gcstools/main.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package main
 
 import (
@@ -7,7 +9,8 @@ import (
 )
 
 var commands = map[string]func(){
-	"generichook": genericHookMain,
+	"generichook":     genericHookMain,
+	"install-drivers": installDriversMain,
 }
 
 func main() {

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.6
 	github.com/google/go-containerregistry v0.5.1
+	github.com/google/uuid v1.3.0
 	github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3
 	github.com/mattn/go-shellwords v1.0.6
 	github.com/opencontainers/runc v1.0.2

go.sum

@@ -365,6 +365,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=

internal/devices/assigned_devices.go

@@ -5,9 +5,6 @@ package devices
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"net"
-	"strings"
 
 	"github.com/Microsoft/hcsshim/internal/cmd"
 	"github.com/Microsoft/hcsshim/internal/log"
@@ -107,32 +104,3 @@ func createDeviceUtilChildrenCommand(deviceUtilPath string, vmBusInstanceID stri
 	args := []string{deviceUtilPath, "children", parentIDsFlag, "--property=location"}
 	return args
 }
-
-// readCsPipeOutput is a helper function that connects to a listener and reads
-// the connection's comma separated output until done. resulting comma separated
-// values are returned in the `result` param. The `errChan` param is used to
-// propagate an errors to the calling function.
-func readCsPipeOutput(l net.Listener, errChan chan<- error, result *[]string) {
-	defer close(errChan)
-	c, err := l.Accept()
-	if err != nil {
-		errChan <- errors.Wrapf(err, "failed to accept named pipe")
-		return
-	}
-	bytes, err := ioutil.ReadAll(c)
-	if err != nil {
-		errChan <- err
-		return
-	}
-
-	elementsAsString := strings.TrimSuffix(string(bytes), "\n")
-	elements := strings.Split(elementsAsString, ",")
-	*result = append(*result, elements...)
-
-	if len(*result) == 0 {
-		errChan <- errors.Wrapf(err, "failed to get any pipe output")
-		return
-	}
-
-	errChan <- nil
-}

internal/devices/drivers.go

@@ -6,18 +6,23 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/Microsoft/hcsshim/internal/cmd"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/resources"
 	"github.com/Microsoft/hcsshim/internal/uvm"
+	"github.com/pkg/errors"
 )
 
-// InstallWindowsDriver mounts a specified kernel driver using vsmb, then installs it in the UVM.
+// InstallKernelDriver mounts a specified kernel driver, then installs it in the UVM.
 //
 // `driver` is a directory path on the host that contains driver files for standard installation.
+// For windows this means files for pnp installation (.inf, .cat, .sys, .cert files).
+// For linux this means a vhd file that contains the drivers under /lib/modules/`uname -r` for use
+// with depmod and modprobe.
 //
-// Returns a ResourceCloser for the added vsmb share. On failure, the vsmb share will be released,
+// Returns a ResourceCloser for the added mount. On failure, the mounted share will be released,
 // the returned ResourceCloser will be nil, and an error will be returned.
-func InstallWindowsDriver(ctx context.Context, vm *uvm.UtilityVM, driver string) (closer resources.ResourceCloser, err error) {
+func InstallKernelDriver(ctx context.Context, vm *uvm.UtilityVM, driver string) (closer resources.ResourceCloser, err error) {
 	defer func() {
 		if err != nil && closer != nil {
 			// best effort clean up allocated resource on failure
@@ -27,14 +32,62 @@ func InstallWindowsDriver(ctx context.Context, vm *uvm.UtilityVM, driver string)
 			closer = nil
 		}
 	}()
-	options := vm.DefaultVSMBOptions(true)
-	closer, err = vm.AddVSMB(ctx, driver, options)
+	if vm.OS() == "windows" {
+		options := vm.DefaultVSMBOptions(true)
+		closer, err = vm.AddVSMB(ctx, driver, options)
+		if err != nil {
+			return closer, fmt.Errorf("failed to add VSMB share to utility VM for path %+v: %s", driver, err)
+		}
+		uvmPath, err := vm.GetVSMBUvmPath(ctx, driver, true)
+		if err != nil {
+			return closer, err
+		}
+		return closer, execPnPInstallDriver(ctx, vm, uvmPath)
+	}
+	uvmPathForShare := fmt.Sprintf(uvm.LCOWGlobalMountPrefix, vm.UVMMountCounter())
+	scsiCloser, err := vm.AddSCSI(ctx, driver, uvmPathForShare, true, false, []string{}, uvm.VMAccessTypeIndividual)
 	if err != nil {
-		return closer, fmt.Errorf("failed to add VSMB share to utility VM for path %+v: %s", driver, err)
+		return closer, fmt.Errorf("failed to add SCSI disk to utility VM for path %+v: %s", driver, err)
 	}
-	uvmPath, err := vm.GetVSMBUvmPath(ctx, driver, true)
+	return scsiCloser, execModprobeInstallDriver(ctx, vm, uvmPathForShare)
+}
+
+func execModprobeInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir string) error {
+	p, l, err := cmd.CreateNamedPipeListener()
 	if err != nil {
-		return closer, err
+		return err
+	}
+	defer l.Close()
+
+	var pipeResults []string
+	errChan := make(chan error)
+
+	go readCsPipeOutput(l, errChan, &pipeResults)
+
+	args := []string{
+		"/bin/install-drivers",
+		driverDir,
+	}
+	req := &cmd.CmdProcessRequest{
+		Args:   args,
+		Stderr: p,
+	}
+
+	exitCode, err := cmd.ExecInUvm(ctx, vm, req)
+	if err != nil && err != noExecOutputErr {
+		return errors.Wrapf(err, "failed to install driver %s in uvm with exit code %d", driverDir, exitCode)
 	}
-	return closer, execPnPInstallDriver(ctx, vm, uvmPath)
+
+	// wait to finish parsing stdout results
+	select {
+	case err := <-errChan:
+		if err != nil {
+			return err
+		}
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	log.G(ctx).WithField("added drivers", driverDir).Debug("installed drivers")
+	return nil
 }

internal/devices/pnp.go

@@ -5,6 +5,9 @@ package devices
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
+	"net"
+	"strings"
 
 	"github.com/Microsoft/hcsshim/internal/cmd"
 	"github.com/Microsoft/hcsshim/internal/log"
@@ -22,6 +25,8 @@ const (
 										is an expected race and can be ignored.`
 )
 
+var noExecOutputErr = errors.New("failed to get any pipe output")
+
 // createPnPInstallDriverCommand creates a pnputil command to add and install drivers
 // present in `driverUVMPath` and all subdirectories.
 func createPnPInstallDriverCommand(driverUVMPath string) []string {
@@ -61,3 +66,32 @@ func execPnPInstallDriver(ctx context.Context, vm *uvm.UtilityVM, driverDir stri
 	log.G(ctx).WithField("added drivers", driverDir).Debug("installed drivers")
 	return nil
 }
+
+// readCsPipeOutput is a helper function that connects to a listener and reads
+// the connection's comma separated output until done. resulting comma separated
+// values are returned in the `result` param. The `errChan` param is used to
+// propagate an errors to the calling function.
+func readCsPipeOutput(l net.Listener, errChan chan<- error, result *[]string) {
+	defer close(errChan)
+	c, err := l.Accept()
+	if err != nil {
+		errChan <- errors.Wrapf(err, "failed to accept named pipe")
+		return
+	}
+	bytes, err := ioutil.ReadAll(c)
+	if err != nil {
+		errChan <- err
+		return
+	}
+
+	elementsAsString := strings.TrimSuffix(string(bytes), "\n")
+	elements := strings.Split(elementsAsString, ",")
+	*result = append(*result, elements...)
+
+	if len(*result) == 0 {
+		errChan <- noExecOutputErr
+		return
+	}
+
+	errChan <- nil
+}

internal/guest/runtime/hcsv2/spec.go

@@ -254,3 +254,58 @@ func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error {
 
 	return nil
 }
+
+// Helper function to create an oci prestart hook to run ldconfig
+func addLDConfigHook(ctx context.Context, spec *oci.Spec, args, env []string) error {
+	if spec.Hooks == nil {
+		spec.Hooks = &oci.Hooks{}
+	}
+
+	ldConfigHook := oci.Hook{
+		Path: "/sbin/ldconfig",
+		Args: args,
+		Env:  env,
+	}
+
+	spec.Hooks.Prestart = append(spec.Hooks.Prestart, ldConfigHook)
+	return nil
+}
+
+func addLinuxDeviceToSpec(ctx context.Context, hostDevice *devices.Device, spec *oci.Spec, addCgroupDevice bool) {
+	rd := oci.LinuxDevice{
+		Path:  hostDevice.Path,
+		Type:  string(hostDevice.Type),
+		Major: hostDevice.Major,
+		Minor: hostDevice.Minor,
+		UID:   &hostDevice.Uid,
+		GID:   &hostDevice.Gid,
+	}
+	if hostDevice.Major == 0 && hostDevice.Minor == 0 {
+		// Invalid device, most likely a symbolic link, skip it.
+		return
+	}
+	found := false
+	for i, dev := range spec.Linux.Devices {
+		if dev.Path == rd.Path {
+			found = true
+			spec.Linux.Devices[i] = rd
+			break
+		}
+		if dev.Type == rd.Type && dev.Major == rd.Major && dev.Minor == rd.Minor {
+			log.G(ctx).Warnf("The same type '%s', major '%d' and minor '%d', should not be used for multiple devices.", dev.Type, dev.Major, dev.Minor)
+		}
+	}
+	if !found {
+		spec.Linux.Devices = append(spec.Linux.Devices, rd)
+		if addCgroupDevice {
+			deviceCgroup := oci.LinuxDeviceCgroup{
+				Allow:  true,
+				Type:   string(hostDevice.Type),
+				Major:  &hostDevice.Major,
+				Minor:  &hostDevice.Minor,
+				Access: string(hostDevice.Permissions),
+			}
+			spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, deviceCgroup)
+		}
+	}
+}