Merge pull request #79 from cybozu-go/snapshot

Support for node snapshot

docs/pmctl.md

@@ -320,6 +320,33 @@ Clear the effect by "delay" and "loss" action.
 $ pmctl net action clear $DEVICE
 ```
 
+`snapshot` subcommand
+----------------
+
+### `pmctl snapshot save TAG`
+
+Save a snapshot of the all VMs as the 'TAG'.
+
+If specify the same tag as before, the snapshot will be overwritten.
+
+NOTE: To save a snapshot, localds and vvfat devices have to be detached.
+
+```console
+$ pmctl snapshot save test
+```
+
+### `pmctl snapshot load TAG`
+
+Restore all VMs from snapshot specified by the 'TAG'.
+
+If there is no snapshot of the tag, restoration is not done, but not reported.
+
+NOTE: To load a snapshot, localds and vvfat devices have to be detached.
+
+```console
+$ pmctl snapshot load test
+```
+
 `completion` subcommand
 -----------------------
 

mtest/example_test.go

@@ -31,6 +31,14 @@ var _ = Describe("example launch test", func() {
 				return nil
 			}).Should(Succeed())
 		})
+		By("saving a snapshot", func() {
+			_, err := pmctl("snapshot", "save", "test")
+			Expect(err).NotTo(HaveOccurred())
+		})
+		By("loading a snapshot", func() {
+			_, err := pmctl("snapshot", "load", "test")
+			Expect(err).NotTo(HaveOccurred())
+		})
 		By("terminate placemat", func() {
 			terminatePlacemat(session)
 			Eventually(session.Exited).Should(BeClosed())

node.go

@@ -2,22 +2,17 @@ package placemat
 
 import (
 	"context"
+	"crypto/sha1"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
-	"path/filepath"
-
-	"crypto/sha1"
-	"math/rand"
-
 	"github.com/cybozu-go/log"
 	"github.com/cybozu-go/well"
 )
@@ -227,7 +222,7 @@ func (n *Node) Start(ctx context.Context, r *Runtime, nodeCh chan<- bmcInfo) (*N
 		devParams := []string{
 			"virtio-net-pci",
 			fmt.Sprintf("netdev=%s", br.Name),
-			fmt.Sprintf("mac=%s", generateRandomMACForKVM()),
+			fmt.Sprintf("mac=%s", generateMACForKVM(n.Name)),
 		}
 		if n.UEFI {
 			// disable iPXE boot
@@ -298,14 +293,6 @@ func (n *Node) Start(ctx context.Context, r *Runtime, nodeCh chan<- bmcInfo) (*N
 		}
 	}
 
-	connMonitor, err := net.Dial("unix", monitor)
-	if err != nil {
-		return nil, err
-	}
-	go func() {
-		io.Copy(ioutil.Discard, connMonitor)
-	}()
-
 	connGuest, err := net.Dial("unix", guest)
 	if err != nil {
 		return nil, err
@@ -320,25 +307,23 @@ func (n *Node) Start(ctx context.Context, r *Runtime, nodeCh chan<- bmcInfo) (*N
 	cleanup := func() {
 		connGuest.Close()
 		os.Remove(guest)
-		connMonitor.Close()
 		os.Remove(monitor)
 		os.Remove(r.socketPath(n.Name))
 	}
 
 	vm := &NodeVM{
 		cmd:     qemuCommand,
-		monitor: connMonitor,
+		monitor: monitor,
 		running: true,
 		cleanup: cleanup,
 	}
 
 	return vm, err
 }
 
-func generateRandomMACForKVM() string {
+func generateMACForKVM(name string) string {
 	vendorPrefix := "52:54:00" // QEMU's vendor prefix
-	bytes := make([]byte, 3)
-	rand.Read(bytes)
+	bytes := sha1.Sum([]byte(name))
 	return fmt.Sprintf("%s:%02x:%02x:%02x", vendorPrefix, bytes[0], bytes[1], bytes[2])
 }
 

node_test.go

@@ -6,16 +6,15 @@ import (
 )
 
 func TestGenerateRandomMacForKVM(t *testing.T) {
-	sut := generateRandomMACForKVM()
+	sut := generateMACForKVM("test")
 	if len(sut) != 17 {
 		t.Fatal("length of MAC address string is not 17")
 	}
-	if sut == generateRandomMACForKVM() {
+	if sut == generateMACForKVM("hoge") {
 		t.Fatal("it should generate unique address")
 	}
 	_, err := net.ParseMAC(sut)
 	if err != nil {
 		t.Fatal("invalid MAC address", err)
 	}
-
 }

node_vm.go

@@ -1,16 +1,25 @@
 package placemat
 
 import (
+	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 
+	"github.com/cybozu-go/log"
 	"github.com/cybozu-go/well"
 )
 
+const (
+	stopCommand   = "stop\n"
+	resumeCommand = "cont\n"
+)
+
 // NodeVM holds resources to manage and monitor a QEMU process.
 type NodeVM struct {
 	cmd     *well.LogCmd
-	monitor net.Conn
+	monitor string
 	running bool
 	cleanup func()
 }
@@ -21,21 +30,189 @@ func (n *NodeVM) IsRunning() bool {
 }
 
 // PowerOn turns on the power of the VM.
-func (n *NodeVM) PowerOn() {
+func (n *NodeVM) PowerOn() error {
 	if n.running {
-		return
+		return nil
+	}
+
+	conn, err := net.Dial("unix", n.monitor)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	go func() {
+		io.Copy(ioutil.Discard, conn)
+	}()
+
+	_, err = io.WriteString(conn, "system_reset\ncont\n")
+	if err != nil {
+		return err
 	}
 
-	io.WriteString(n.monitor, "system_reset\ncont\n")
 	n.running = true
+	return nil
 }
 
 // PowerOff turns off the power of the VM.
-func (n *NodeVM) PowerOff() {
+func (n *NodeVM) PowerOff() error {
 	if !n.running {
-		return
+		return nil
+	}
+
+	conn, err := net.Dial("unix", n.monitor)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	go func() {
+		io.Copy(ioutil.Discard, conn)
+	}()
+
+	_, err = io.WriteString(conn, "stop\n")
+	if err != nil {
+		return err
 	}
 
-	io.WriteString(n.monitor, "stop\n")
 	n.running = false
+	return nil
+}
+
+func execQEMUCommand(ctx context.Context, monitor, cmd string) (string, error) {
+	var d net.Dialer
+	conn, err := d.DialContext(ctx, "unix", monitor)
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close()
+
+	resp := make(chan string)
+	go func() error {
+		buf := make([]byte, 4096)
+		for {
+			n, err := conn.Read(buf[:])
+			if err != nil {
+				return err
+			}
+			resp <- string(buf[0:n])
+			break
+		}
+		return nil
+	}()
+
+	_, err = io.WriteString(conn, cmd)
+	if err != nil {
+		return "", err
+	}
+	result := <-resp
+	return string(result), nil
+}
+
+func removeBlockDevices(ctx context.Context, monitor string, volumes []NodeVolumeSpec) error {
+	for i, v := range volumes {
+		if v.Kind == "localds" || v.Kind == "vvfat" {
+			out, err := execQEMUCommand(ctx, monitor, fmt.Sprintf("drive_del virtio%d\n", i))
+			if err != nil {
+				return err
+			}
+			log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+				"monitor": monitor,
+			})
+		}
+	}
+	return nil
+}
+
+// SaveVM saves a snapshot of the VM. To save a snapshot, localds and vvfat devices have to be detached.
+// NOTE: virtio block device does not support hot add. After saving snapshot, you will no longer access mounted block device other than rootfs.
+//       https://github.com/ceph/qemu-kvm/blob/de4eb6c5347e40b02dbe72cda18b58654ad11242/hw/pci-hotplug.c#L143
+func (n *NodeVM) SaveVM(ctx context.Context, node *Node, tag string) error {
+	if !n.running {
+		return nil
+	}
+
+	// Stop VM
+	out, err := execQEMUCommand(ctx, n.monitor, stopCommand)
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+		"monitor": n.monitor,
+	})
+
+	// Detach localds and vvfat
+	err = removeBlockDevices(ctx, n.monitor, node.Volumes)
+	if err != nil {
+		return err
+	}
+
+	// Save snapshot
+	out, err = execQEMUCommand(ctx, n.monitor, fmt.Sprintf("savevm %s\n", tag))
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+		"monitor": n.monitor,
+	})
+
+	// Resume VM
+	out, err = execQEMUCommand(ctx, n.monitor, resumeCommand)
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+		"monitor": n.monitor,
+	})
+
+	return nil
+}
+
+// LoadVM loads a snapshot of the VM. To load a snapshot, localds and vvfat devices have to be detached.
+// NOTE: virtio block device does not support hot add. After loading snapshot, you will no longer access mounted block device other than rootfs.
+//       https://github.com/ceph/qemu-kvm/blob/de4eb6c5347e40b02dbe72cda18b58654ad11242/hw/pci-hotplug.c#L143
+func (n *NodeVM) LoadVM(ctx context.Context, node *Node, tag string) error {
+	if !n.running {
+		return nil
+	}
+
+	// Stop VM
+	out, err := execQEMUCommand(ctx, n.monitor, stopCommand)
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+		"monitor": n.monitor,
+	})
+
+	// Remove block devices
+	for i, v := range node.Volumes {
+		if v.Kind == "localds" || v.Kind == "vvfat" {
+			out, err := execQEMUCommand(ctx, n.monitor, fmt.Sprintf("drive_del virtio%d\n", i))
+			if err != nil {
+				return err
+			}
+			log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+				"monitor": n.monitor,
+			})
+		}
+	}
+
+	// Load snapshot
+	out, err = execQEMUCommand(ctx, n.monitor, fmt.Sprintf("loadvm %s\n", tag))
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+		"monitor": n.monitor,
+	})
+
+	// Resume VM
+	out, err = execQEMUCommand(ctx, n.monitor, resumeCommand)
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("monitor log: %s", out), map[string]interface{}{
+		"monitor": n.monitor,
+	})
+
+	return nil
 }

pkg/pmctl/cmd/snapshot.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// snapshotCmd represents the node command
+var snapshotCmd = &cobra.Command{
+	Use:   "snapshot",
+	Short: "snapshot subcommand",
+	Long:  `snapshot subcommand is the parent of commands that control snapshots`,
+}
+
+func init() {
+	rootCmd.AddCommand(snapshotCmd)
+}