diff --git a/README.md b/README.md new file mode 100644 index 00000000..535752fb --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +[![Go Report Card](https://goreportcard.com/badge/github.com/sameo/virtcontainers)](https://goreportcard.com/report/github.com/sameo/virtcontainers) + +# VirtContainers + +VirtContainers is a Go package for building hardware virtualized container runtimes. + +## Scope + +VirtContainers is not a container runtime implementation, but aims at factorizing +hardware virtualization code in order to build VM based container runtimes. + +The few existing VM based container runtimes (Clear Containers, RunV, Rkt +kvm stage 1) all share the same hardware virtualization semantics but use different +code bases to implement them. VirtContainers goal is to factorize this code into +a common Go library. + +Ideally VM based container runtime implementations would become translation layers +from the runtime specification they implement to the VirtContainers API. + +## Out of scope + +Implementing yet another container runtime is out of VirtContainers scope. Any tools +or executables provided with VirtContainers are only provided for demonstration or +testing purposes. + +## Design + +### Goals + +VirtContainers is a container specification agnostic Go package and thus tries to +abstract the various container runtime specifications (OCI, AppC and CRI) and present +that as its high level API. + +### Pods + +The VirtContainers execution unit is a Pod, i.e. VirtContainers callers start pods +where containers will be running. + +Virtcontainers creates a pod by starting a virtual machine and setting the pod up within +that environment. Starting a pod means launching all containers with the VM pod runtime +environment. + +### Hypervisors + +The virtcontainers package relies on hypervisors to start and stop virtual machine where +pods will be running. An hypervisor is defined by an Hypervisor interface implementation, +and the default implementation is the QEMU one. + +### Agents + +During the lifecycle of a container, the runtime running on the host needs to interact with +the virtual machine guest OS in order to start new commands to be executed as part of a given +container workload, set new networking routes or interfaces, fetch a container standard or +error output, and so on. +There are many existing and potential solutions to resolve that problem and virtcontainers abstract +this through the Agent interface. + +## API + +The high level VirtContainers API is the following one: + +### Pod API + +* `CreatePod(podConfig PodConfig)` creates a Pod. +The Pod is prepared and will run into a virtual machine. It is not started, i.e. the VM is not running after `CreatePod()` is called. + +* `DeletePod(podID string)` deletes a Pod. +The function will fail if the Pod is running. In that case `StopPod()` needs to be called first. + +* `StartPod(podID string)` starts an already created Pod. + +* `StopPod(podID string)` stops an already running Pod. + +* `ListPod()` lists all running Pods on the host. + +* `EnterPod(cmd Cmd)` enters a Pod root filesystem and runs a given command. + +* `PodStatus(podID string)` returns a detailed Pod status. + +### Container API + +* `CreateContainer(podID string, container ContainerConfig)` creates a Container on a given Pod. + +* `DeleteContainer(containerID string)` deletes a Container from a Pod. If the container is running it needs to be stopped first. + +* `StartContainer(containerID string)` starts an already created container. + +* `StopContainer(containerID string)` stops an already running container. + +* `EnterContainer(containerID string, cmd Cmd)` enters an already running container and runs a given command. + +* `ContainerStatus(containerID string)` returns a detailed container status. diff --git a/agent.go b/agent.go new file mode 100644 index 00000000..710cad38 --- /dev/null +++ b/agent.go @@ -0,0 +1,130 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "fmt" + + "github.com/mitchellh/mapstructure" +) + +// AgentType describes the type of guest agent a Pod should run. +type AgentType string + +const ( + // NoopAgentType is the No-Op agent. + NoopAgentType AgentType = "noop" + + // SSHdAgent is the SSH daemon agent. + SSHdAgent = "sshd" + + // HyperstartAgent is the Hyper hyperstart agent. + HyperstartAgent = "hyperstart" +) + +// Set sets an agent type based on the input string. +func (agentType *AgentType) Set(value string) error { + switch value { + case "noop": + *agentType = NoopAgentType + return nil + case "sshd": + *agentType = SSHdAgent + return nil + case "hyperstart": + *agentType = HyperstartAgent + return nil + default: + return fmt.Errorf("Unknown agent type %s", value) + } +} + +// String converts an agent type to a string. +func (agentType *AgentType) String() string { + switch *agentType { + case NoopAgentType: + return string(NoopAgentType) + case SSHdAgent: + return string(SSHdAgent) + case HyperstartAgent: + return string(HyperstartAgent) + default: + return "" + } +} + +// newAgent returns an agent from an agent type. +func newAgent(agentType AgentType) (agent, error) { + switch agentType { + case NoopAgentType: + return &noopAgent{}, nil + case SSHdAgent: + return &sshd{}, nil + case HyperstartAgent: + return &hyper{}, nil + default: + return &noopAgent{}, nil + } +} + +// newAgentConfig returns an agent config from a generic PodConfig interface. +func newAgentConfig(config PodConfig) interface{} { + switch config.AgentType { + case NoopAgentType: + return nil + case SSHdAgent: + var sshdConfig SshdConfig + err := mapstructure.Decode(config.AgentConfig, &sshdConfig) + if err != nil { + return err + } + return sshdConfig + case HyperstartAgent: + var hyperConfig HyperConfig + err := mapstructure.Decode(config.AgentConfig, &hyperConfig) + if err != nil { + return err + } + return hyperConfig + default: + return nil + } +} + +// Agent is the virtcontainers agent interface. +// Agents are running in the guest VM and handling +// communications between the host and guest. +type agent interface { + // init is used to pass agent specific configuration to the agent implementation. + // agent implementations also will typically start listening for agent events from + // init(). + // After init() is called, agent implementations should be initialized and ready + // to handle all other Agent interface methods. + init(config interface{}, hypervisor hypervisor) error + + // start will start the agent on the host. + start() error + + // exec will tell the agent to run a command in an already running container. + exec(podID string, contID string, cmd Cmd) error + + // StartPod will tell the agent to start all containers related to the Pod. + startPod(config PodConfig) error + + // StopPod will tell the agent to stop all containers related to the Pod. + stopPod(config PodConfig) error +} diff --git a/api.go b/api.go new file mode 100644 index 00000000..1ca35e37 --- /dev/null +++ b/api.go @@ -0,0 +1,164 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "fmt" + "os" + "text/tabwriter" +) + +// CreatePod is the virtcontainers pod creation entry point. +// CreatePod creates a pod and its containers. It does not start them. +func CreatePod(podConfig PodConfig) (*Pod, error) { + // Create the pod. + p, err := createPod(podConfig) + if err != nil { + return nil, err + } + + // Store it. + err = p.storePod() + if err != nil { + return nil, err + } + + return p, nil +} + +// DeletePod is the virtcontainers pod deletion entry point. +// DeletePod will stop an already running container and then delete it. +func DeletePod(podID string) (*Pod, error) { + // Fetch the pod from storage and create it. + p, err := fetchPod(podID) + if err != nil { + return nil, err + } + + // Delete it. + err = p.delete() + if err != nil { + return nil, err + } + + return p, nil +} + +// StartPod is the virtcontainers pod starting entry point. +// StartPod will talk to the given hypervisor to start an existing +// pod and all its containers. +// It returns the pod ID. +func StartPod(podID string) (*Pod, error) { + // Fetch the pod from storage and create it. + p, err := fetchPod(podID) + if err != nil { + return nil, err + } + + // Start it. + err = p.start() + if err != nil { + p.delete() + return nil, err + } + + return p, nil +} + +// StopPod is the virtcontainers pod stopping entry point. +// StopPod will talk to the given agent to stop an existing pod and destroy all containers within that pod. +func StopPod(podID string) (*Pod, error) { + // Fetch the pod from storage and create it. + p, err := fetchPod(podID) + if err != nil { + return nil, err + } + + // Stop it. + err = p.stop() + if err != nil { + p.delete() + return nil, err + } + + return p, nil +} + +// RunPod is the virtcontainers pod running entry point. +// RunPod creates a pod and its containers and then it starts them. +func RunPod(podConfig PodConfig) (*Pod, error) { + // Create the pod. + p, err := createPod(podConfig) + if err != nil { + return nil, err + } + + // Store it. + err = p.storePod() + if err != nil { + return nil, err + } + + // Start it. + err = p.start() + if err != nil { + p.delete() + return nil, err + } + + return p, nil +} + +var listFormat = "%s\t%s\t%s\t%s\n" + +// ListPod is the virtcontainers pod listing entry point. +func ListPod() error { + dir, err := os.Open(configStoragePath) + if err != nil { + return err + } + + defer dir.Close() + + pods, err := dir.Readdirnames(0) + if err != nil { + return err + } + + fs := filesystem{} + + w := tabwriter.NewWriter(os.Stdout, 2, 8, 1, '\t', 0) + fmt.Fprintf(w, listFormat, "POD ID", "STATE", "HYPERVISOR", "AGENT") + + for _, p := range pods { + config, err := fs.fetchConfig(p) + if err != nil { + continue + } + + state, err := fs.fetchState(p) + if err != nil { + continue + } + + fmt.Fprintf(w, listFormat, + config.ID, state.State, config.HypervisorType, config.AgentType) + } + + w.Flush() + return nil +} diff --git a/hyperstart.go b/hyperstart.go new file mode 100644 index 00000000..679db8a5 --- /dev/null +++ b/hyperstart.go @@ -0,0 +1,429 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "encoding/json" + "fmt" + "math/rand" + "net" + "strconv" + "time" + + "github.com/golang/glog" + + hyperJson "github.com/hyperhq/runv/hyperstart/api/json" +) + +// Control command IDs +// Need to be in sync with hyperstart/src/api.h +const ( + getVersion = "\x00\x00\x00\x00" + startPod = "\x00\x00\x00\x01" + getPod = "\x00\x00\x00\x02" + stopPodDeprecated = "\x00\x00\x00\x03" + destroyPod = "\x00\x00\x00\x04" + restartContainer = "\x00\x00\x00\x05" + execCommand = "\x00\x00\x00\x06" + cmdFinished = "\x00\x00\x00\x07" + ready = "\x00\x00\x00\x08" + ack = "\x00\x00\x00\x09" + hyperError = "\x00\x00\x00\x0a" + winSize = "\x00\x00\x00\x0b" + ping = "\x00\x00\x00\x0c" + podFinished = "\x00\x00\x00\x0d" + next = "\x00\x00\x00\x0e" + writeFile = "\x00\x00\x00\x0f" + readFile = "\x00\x00\x00\x10" + newContainer = "\x00\x00\x00\x11" + killContainer = "\x00\x00\x00\x12" + onlineCPUMem = "\x00\x00\x00\x13" + setupInterface = "\x00\x00\x00\x14" + setupRoute = "\x00\x00\x00\x15" + removeContainer = "\x00\x00\x00\x16" +) + +// Values related to the communication on control channel. +const ( + ctlHdrSize = 8 + ctlHdrLenOffset = 4 +) + +// Values related to the communication on tty channel. +const ( + ttyHdrSize = 12 + ttyHdrLenOffset = 8 +) + +// chType differentiates channels type. +type chType uint8 + +// List of possible values for channels type. +const ( + ctlType chType = iota + ttyType +) + +// HyperConfig is a structure storing information needed for +// hyperstart agent initialization. +type HyperConfig struct { + SockCtlName string + SockTtyName string + SockCtlType string + SockTtyType string +} + +// hyper is the Agent interface implementation for hyperstart. +type hyper struct { + config HyperConfig + hypervisor hypervisor + + sharedDir Volume + + cCtl net.Conn + cTty net.Conn +} + +// frame is the structure corresponding to the frame format +// used to send and receive on different channels. +type frame struct { + cmd string + payloadLen string + payload string +} + +// execInfo is the structure corresponding to the format +// expected by hyperstart to execute a command on the guest. +type execInfo struct { + container string + process hyperJson.Process +} + +func (c HyperConfig) validate() bool { + return true +} + +func send(c net.Conn, frame frame) error { + strArray := frame.cmd + frame.payloadLen + frame.payload + + c.Write([]byte(strArray)) + + return nil +} + +func recv(c net.Conn, chType chType) (frame, error) { + var frame frame + var hdrSize int + var hdrLenOffset int + + switch chType { + case ctlType: + hdrSize = ctlHdrSize + hdrLenOffset = ctlHdrLenOffset + case ttyType: + hdrSize = ttyHdrSize + hdrLenOffset = ttyHdrLenOffset + } + + byteHdr := make([]byte, hdrSize) + + byteRead, err := c.Read(byteHdr) + if err != nil { + return frame, err + } + + glog.Infof("Header received: %x\n", byteHdr) + + if byteRead != hdrSize { + return frame, fmt.Errorf("Not enough bytes read (%d/%d)\n", byteRead, hdrSize) + } + + frame.cmd = string(byteHdr[:hdrLenOffset]) + frame.payloadLen = string(byteHdr[hdrLenOffset:]) + + payloadLen, err := strconv.ParseUint(fmt.Sprintf("%x", frame.payloadLen), 16, 0) + if err != nil { + return frame, err + } + + payloadLen -= uint64(hdrSize) + glog.Infof("Payload length: %d\n", payloadLen) + + if payloadLen == 0 { + return frame, nil + } + + bytePayload := make([]byte, payloadLen) + + byteRead, err = c.Read(bytePayload) + if err != nil { + return frame, err + } + + glog.Infof("Payload received: %x\n", bytePayload) + if chType == ttyType { + glog.Infof("String formatted payload: %s\n", string(bytePayload)) + } + + if byteRead != int(payloadLen) { + return frame, fmt.Errorf("Not enough bytes read (%d/%d)\n", byteRead, payloadLen) + } + + frame.payload = string(bytePayload) + + return frame, nil +} + +func waitForReply(c net.Conn, cmdID string) error { + for { + frame, err := recv(c, ctlType) + if err != nil { + return err + } + + if frame.cmd == cmdID { + break + } + + if frame.cmd == next || frame.cmd == ready { + continue + } + + if frame.cmd != cmdID { + if frame.cmd == hyperError { + return fmt.Errorf("ERROR received from Hyperstart\n") + } + + return fmt.Errorf("CMD ID received %x not matching expected %x\n", frame.cmd, cmdID) + } + } + + return nil +} + +func sendCmd(c net.Conn, cmdID string, payload interface{}) error { + var payloadStr string + + if payload != nil { + jsonOut, err := json.Marshal(payload) + if err != nil { + return err + } + + payloadStr = string(jsonOut) + } else { + payloadStr = "" + } + + glog.Infof("payload: %s\n", payloadStr) + intLen := len(payloadStr) + ctlHdrSize + payloadLen := intTo4BytesString(intLen) + glog.Infof("payload len: %x\n", payloadLen) + + frame := frame{ + cmd: cmdID, + payloadLen: payloadLen, + payload: payloadStr, + } + + err := send(c, frame) + if err != nil { + return err + } + + if cmdID == destroyPod { + return nil + } + + err = waitForReply(c, ack) + if err != nil { + return err + } + + return nil +} + +func intTo4BytesString(val int) string { + var buf [4]byte + + buf[0] = byte(val >> 24) + buf[1] = byte(val >> 16) + buf[2] = byte(val >> 8) + buf[3] = byte(val) + + return string(buf[:4]) +} + +func retryConnectSocket(retry int, sockType, sockName string) (net.Conn, error) { + var err error + var c net.Conn + + for i := 0; i < retry; i++ { + c, err = net.Dial(sockType, sockName) + if err == nil { + break + } + + select { + case <-time.After(100 * time.Millisecond): + break + } + } + + if err != nil { + return nil, fmt.Errorf("Failed to dial on %s socket %s: %s\n", sockType, sockName, err) + } + + return c, nil +} + +func buildHyperContainerProcess(cmd Cmd) (hyperJson.Process, error) { + var envVars []hyperJson.EnvironmentVar + + for _, e := range cmd.Envs { + envVar := hyperJson.EnvironmentVar{ + Env: e.Var, + Value: e.Value, + } + + envVars = append(envVars, envVar) + } + + process := hyperJson.Process{ + User: cmd.User, + Group: cmd.Group, + Stdio: uint64(rand.Int63()), + Stderr: uint64(rand.Int63()), + Args: cmd.Args, + Envs: envVars, + Workdir: cmd.WorkDir, + } + + return process, nil +} + +// init is the agent initialization implementation for hyperstart. +func (h *hyper) init(config interface{}, hypervisor hypervisor) error { + switch c := config.(type) { + case HyperConfig: + if c.validate() == false { + return fmt.Errorf("Invalid configuration\n") + } + h.config = c + default: + return fmt.Errorf("Invalid config type\n") + } + + h.hypervisor = hypervisor + + h.sharedDir = Volume{ + MountTag: "shared", + HostPath: "/", + } + + err := h.hypervisor.addDevice(h.sharedDir, fsDev) + if err != nil { + return err + } + + return nil +} + +// start is the agent starting implementation for hyperstart. +func (h *hyper) start() error { + var err error + + h.cCtl, err = retryConnectSocket(1000, h.config.SockCtlType, h.config.SockCtlName) + if err != nil { + return err + } + + h.cTty, err = retryConnectSocket(1000, h.config.SockTtyType, h.config.SockTtyName) + if err != nil { + return err + } + + err = sendCmd(h.cCtl, ping, nil) + if err != nil { + return err + } + + return nil +} + +// exec is the agent command execution implementation for hyperstart. +func (h *hyper) exec(podID string, contID string, cmd Cmd) error { + process, err := buildHyperContainerProcess(cmd) + if err != nil { + return err + } + + execInfo := execInfo{ + container: contID, + process: process, + } + + err = sendCmd(h.cCtl, execCommand, execInfo) + if err != nil { + return err + } + + return nil +} + +// startPod is the agent Pod starting implementation for hyperstart. +func (h *hyper) startPod(config PodConfig) error { + var containers []hyperJson.Container + + for _, c := range config.Containers { + process, err := buildHyperContainerProcess(c.Cmd) + if err != nil { + return err + } + + container := hyperJson.Container{ + Id: c.ID, + Rootfs: c.RootFs, + Process: process, + } + + containers = append(containers, container) + } + + hyperPod := hyperJson.Pod{ + Hostname: config.ID, + Containers: containers, + ShareDir: h.sharedDir.MountTag, + } + + err := sendCmd(h.cCtl, startPod, hyperPod) + if err != nil { + return err + } + + return nil +} + +// stopPod is the agent Pod stopping implementation for hyperstart. +func (h *hyper) stopPod(config PodConfig) error { + err := sendCmd(h.cCtl, destroyPod, nil) + if err != nil { + return err + } + + return nil +} diff --git a/hypervisor.go b/hypervisor.go new file mode 100644 index 00000000..5729952c --- /dev/null +++ b/hypervisor.go @@ -0,0 +1,146 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "fmt" +) + +// HypervisorType describes an hypervisor type. +type HypervisorType string + +const ( + // QemuHypervisor is the QEMU hypervisor. + QemuHypervisor HypervisorType = "qemu" +) + +// deviceType describes a virtualized device type. +type deviceType int + +const ( + // ImgDev is the image device type. + imgDev deviceType = iota + + // FsDev is the filesystem device type. + fsDev + + // NetDev is the network device type. + netDev + + // SerialDev is the serial device type. + serialDev + + // BlockDev is the block device type. + blockDev + + // ConsoleDev is the console device type. + consoleDev +) + +// Set sets an hypervisor type based on the input string. +func (hType *HypervisorType) Set(value string) error { + switch value { + case "qemu": + *hType = QemuHypervisor + return nil + default: + return fmt.Errorf("Unknown hypervisor type %s", value) + } +} + +// String converts an hypervisor type to a string. +func (hType *HypervisorType) String() string { + switch *hType { + case QemuHypervisor: + return string(QemuHypervisor) + default: + return "" + } +} + +// newHypervisor returns an hypervisor from and hypervisor type. +func newHypervisor(hType HypervisorType) (hypervisor, error) { + switch hType { + case QemuHypervisor: + return &qemu{}, nil + default: + return nil, fmt.Errorf("Unknown hypervisor type %s", hType) + } +} + +// Param is a key/value representation for hypervisor and kernel parameters. +type Param struct { + parameter string + value string +} + +// HypervisorConfig is the hypervisor configuration. +type HypervisorConfig struct { + // KernelPath is the guest kernel host path. + KernelPath string + + // ImagePath is the guest image host path. + ImagePath string + + // HypervisorPath is the hypervisor executable host path. + HypervisorPath string + + // KernelParams are additional guest kernel parameters. + KernelParams []Param + + // HypervisorParams are additional hypervisor parameters. + HypervisorParams []Param +} + +func (conf *HypervisorConfig) validate() bool { + return true +} + +func appendParam(params []Param, parameter string, value string) []Param { + return append(params, Param{parameter, value}) +} + +func serializeParams(params []Param, delim string) []string { + var parameters []string + + for _, p := range params { + if p.parameter == "" && p.value == "" { + continue + } else if p.parameter == "" { + parameters = append(parameters, fmt.Sprintf("%s", p.value)) + } else if p.value == "" { + parameters = append(parameters, fmt.Sprintf("%s", p.parameter)) + } else if delim == "" { + parameters = append(parameters, fmt.Sprintf("%s", p.parameter)) + parameters = append(parameters, fmt.Sprintf("%s", p.value)) + } else { + parameters = append(parameters, fmt.Sprintf("%s%s%s", p.parameter, delim, p.value)) + } + } + + return parameters +} + +// hypervisor is the virtcontainers hypervisor interface. +// The default hypervisor implementation is Qemu. +type hypervisor interface { + init(config HypervisorConfig) error + createPod(podConfig PodConfig) error + startPod(startCh, stopCh chan struct{}) error + stopPod() error + addDevice(devInfo interface{}, devType deviceType) error +} diff --git a/noop_agent.go b/noop_agent.go new file mode 100644 index 00000000..0c6479b6 --- /dev/null +++ b/noop_agent.go @@ -0,0 +1,47 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +// noopAgent a.k.a. NO-OP Agent is an empty Agent implementation, for testing and +// mocking purposes. +type noopAgent struct { +} + +// init initializes the Noop agent, i.e. it does nothing. +func (n *noopAgent) init(config interface{}, hypervisor hypervisor) error { + return nil +} + +// start is the Noop agent starting implementation. It does nothing. +func (n *noopAgent) start() error { + return nil +} + +// exec is the Noop agent command execution implementation. It does nothing. +func (n *noopAgent) exec(podID string, contID string, cmd Cmd) error { + return nil +} + +// startPod is the Noop agent Pod starting implementation. It does nothing. +func (n *noopAgent) startPod(config PodConfig) error { + return nil +} + +// stopPod is the Noop agent Pod stopping implementation. It does nothing. +func (n *noopAgent) stopPod(config PodConfig) error { + return nil +} diff --git a/nsenter.go b/nsenter.go new file mode 100644 index 00000000..9c9f648e --- /dev/null +++ b/nsenter.go @@ -0,0 +1,40 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +// nsenter is a spawner implementation for the nsenter util-linux command. +type nsenter struct { + ContConfig ContainerConfig +} + +const ( + // NsenterCmd is the command used to start nsenter. + nsenterCmd = "nsenter" +) + +// formatArgs is the spawner command formatting implementation for nsenter. +func (n *nsenter) formatArgs(args []string) ([]string, error) { + var newArgs []string + pid := "-1" + + // TODO: Retrieve container PID from container ID + + newArgs = append(newArgs, nsenterCmd+" --target "+pid+" --mount --uts --ipc --net --pid") + newArgs = append(newArgs, args...) + + return newArgs, nil +} diff --git a/pod.go b/pod.go new file mode 100644 index 00000000..4b11672e --- /dev/null +++ b/pod.go @@ -0,0 +1,809 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/01org/ciao/ssntp/uuid" + "github.com/golang/glog" +) + +// controlSocket is the pod control socket. +// It is an hypervisor resource, and for example qemu's control +// socket is the QMP one. +const controlSocket = "ctrl.sock" + +// monitorSocket is the pod monitoring socket. +// It is an hypervisor resource, and is a qmp socket in the qemu case. +// This is a socket that any monitoring entity will listen to in order +// to understand if the VM is still alive or not. +const monitorSocket = "monitor.sock" + +// podResource is an int representing a pod resource type +type podResource int + +const ( + // configFileType represents a configuration file type + configFileType podResource = iota + + // stateFileType represents a state file type + stateFileType + + // lockFileType represents a lock file type + lockFileType +) + +// configStoragePath is the pod configuration directory. +// It will contain one config.json file for each created pod. +const configStoragePath = "/var/lib/virtcontainers/pods" + +// runStoragePath is the pod runtime directory. +// It will contain one state.json and one lock file for each created pod. +const runStoragePath = "/run/virtcontainers/pods" + +// configFile is the file name used for every JSON pod configuration. +const configFile = "config.json" + +// stateFile is the file name storing a pod state. +const stateFile = "state.json" + +// lockFile is the file name locking the usage of a pod. +const lockFile = "lock" + +// stateString is a string representing a pod state. +type stateString string + +const ( + // podReady represents a pod that's ready to be run + podReady stateString = "ready" + + // podRunning represents a pod that's currently running. + podRunning = "running" + + // podPaused represents a pod that has been paused. + podPaused = "paused" +) + +// PodState is a pod state structure. +type PodState struct { + State stateString `json:"state"` +} + +// valid checks that the pod state is valid. +func (state *PodState) valid() bool { + validStates := []stateString{podReady, podRunning, podPaused} + + for _, validState := range validStates { + if state.State == validState { + return true + } + } + + return false +} + +// validTransition returns an error if we want to move to +// an unreachable state. +func (state *PodState) validTransition(oldState stateString, newState stateString) error { + if state.State != oldState { + return fmt.Errorf("Invalid state %s (Expecting %s)", state.State, oldState) + } + + switch state.State { + case podReady: + if newState == podRunning { + return nil + } + + case podRunning: + if newState == podPaused || newState == podReady { + return nil + } + + case podPaused: + if newState == podRunning { + return nil + } + } + + return fmt.Errorf("Can not move from %s to %s", + state.State, newState) +} + +// Volume is a shared volume between the host and the VM, +// defined by its mount tag and its host path. +type Volume struct { + // MountTag is a label used as a hint to the guest. + MountTag string + + // HostPath is the host filesystem path for this volume. + HostPath string +} + +// EnvVar is a key/value structure representing a command +// environment variable. +type EnvVar struct { + Var string + Value string +} + +// Cmd represents a command to execute in a running container. +type Cmd struct { + Args []string + Envs []EnvVar + WorkDir string + + User string + Group string +} + +// ContainerConfig describes one container runtime configuration. +type ContainerConfig struct { + ID string + + // RootFs is the container workload image on the host. + RootFs string + + // Interactive specifies if the container runs in the foreground. + Interactive bool + + // Console is a console path provided by the caller. + Console string + + // Cmd specifies the command to run on a container + Cmd Cmd +} + +// PodConfig is a Pod configuration. +type PodConfig struct { + ID string + + HypervisorType HypervisorType + HypervisorConfig HypervisorConfig + + AgentType AgentType + AgentConfig interface{} + + // Rootfs is the pod root file system in the host. + // This can be left empty if we only have a set of containers + // workload images and expect the agent to aggregate them into + // a pod from the guest. + RootFs string + + // Volumes is a list of shared volumes between the host and the Pod. + Volumes []Volume + + // Containers describe the list of containers within a Pod. + // This list can be empty and populated by adding containers + // to the Pod a posteriori. + Containers []ContainerConfig +} + +// valid checks that the pod configuration is valid. +func (podConfig *PodConfig) valid() bool { + if _, err := newAgent(podConfig.AgentType); err != nil { + glog.Error(err) + return false + } + + if _, err := newHypervisor(podConfig.HypervisorType); err != nil { + podConfig.HypervisorType = QemuHypervisor + } + + if podConfig.ID == "" { + podConfig.ID = uuid.Generate().String() + } + + return true +} + +// PodStorage is the virtcontainers pod storage interface. +// The default pod storage implementation is Filesystem. +type podStorage interface { + storeConfig(config PodConfig) error + fetchConfig(podID string) (PodConfig, error) + storeState(podID string, state PodState) error + fetchState(podID string) (PodState, error) + delete(podID string) error +} + +// Filesystem is a Storage interface implementation. +type filesystem struct { +} + +func podDir(podID string, resource podResource) (string, error) { + var path string + + if podID == "" { + return "", fmt.Errorf("PodID cannot be empty") + } + + switch resource { + case configFileType: + path = configStoragePath + break + case stateFileType, lockFileType: + path = runStoragePath + break + default: + return "", fmt.Errorf("Invalid pod resource") + } + + dirPath := filepath.Join(path, podID) + + return dirPath, nil +} + +func podFile(podID string, resource podResource) (string, error) { + var filename string + + if podID == "" { + return "", fmt.Errorf("PodID cannot be empty") + } + + dirPath, err := podDir(podID, resource) + if err != nil { + return "", err + } + + switch resource { + case configFileType: + filename = configFile + break + case stateFileType: + filename = stateFile + case lockFileType: + filename = lockFile + break + default: + return "", fmt.Errorf("Invalid pod resource") + } + + filePath := filepath.Join(dirPath, filename) + + return filePath, nil +} + +// storeConfig is the storage pod configuration storage implementation for filesystem. +func (fs *filesystem) storeConfig(config PodConfig) error { + if config.valid() == false { + return fmt.Errorf("Invalid pod configuration") + } + + podConfigFile, err := podFile(config.ID, configFileType) + if err != nil { + return err + } + + _, err = os.Stat(podConfigFile) + if err == nil { + os.Remove(podConfigFile) + } + + f, err := os.Create(podConfigFile) + if err != nil { + return err + } + defer f.Close() + + jsonOut, err := json.Marshal(config) + if err != nil { + glog.Errorf("Could not marshall pod config: %s\n", err) + return err + } + f.Write(jsonOut) + + return nil +} + +// fetchConfig is the storage pod configuration retrieval implementation for filesystem. +func (fs *filesystem) fetchConfig(podID string) (PodConfig, error) { + var config PodConfig + + podConfigFile, err := podFile(podID, configFileType) + if err != nil { + return config, err + } + + _, err = os.Stat(podConfigFile) + if err != nil { + return config, err + } + + fileData, err := ioutil.ReadFile(podConfigFile) + if err != nil { + return config, err + } + + err = json.Unmarshal([]byte(string(fileData)), &config) + if err != nil { + return config, err + } + + return config, nil +} + +// storeState is the storage pod state storage implementation for filesystem. +func (fs *filesystem) storeState(podID string, state PodState) error { + if state.valid() == false { + return fmt.Errorf("Invalid pod state") + } + + podStateFile, err := podFile(podID, stateFileType) + if err != nil { + return err + } + + _, err = os.Stat(podStateFile) + if err == nil { + os.Remove(podStateFile) + } + + f, err := os.Create(podStateFile) + if err != nil { + return err + } + defer f.Close() + + jsonOut, err := json.Marshal(state) + if err != nil { + glog.Errorf("Could not marshall pod state: %s\n", err) + return err + } + f.Write(jsonOut) + + return nil +} + +// fetchState is the storage pod state retrieval implementation for filesystem. +func (fs *filesystem) fetchState(podID string) (PodState, error) { + var state PodState + + podStateFile, err := podFile(podID, stateFileType) + if err != nil { + return state, err + } + + _, err = os.Stat(podStateFile) + if err != nil { + return state, err + } + + fileData, err := ioutil.ReadFile(podStateFile) + if err != nil { + return state, err + } + + err = json.Unmarshal([]byte(string(fileData)), &state) + if err != nil { + return state, err + } + + return state, nil +} + +// delete is the storage pod configuration removal implementation for filesystem. +func (fs *filesystem) delete(podID string) error { + resources := []podResource{configFileType, stateFileType} + + for _, resource := range resources { + dir, err := podDir(podID, resource) + if err != nil { + return err + } + + err = os.RemoveAll(dir) + if err != nil { + return err + } + } + + return nil +} + +// Pod is composed of a set of containers and a runtime environment. +// A Pod can be created, deleted, started, stopped, listed, entered, paused and restored. +type Pod struct { + id string + + hypervisor hypervisor + agent agent + storage podStorage + + config *PodConfig + + rootFs string + volumes []Volume + + containers []ContainerConfig + + runPath string + configPath string + + controlSocket string + + state PodState + + lockFile *os.File +} + +// ID returns the pod identifier string. +func (p *Pod) ID() string { + return p.id +} + +// lock locks the current pod to prevent it from being accessed +// by other processes +func (p *Pod) lock() error { + podlockFile, err := podFile(p.id, lockFileType) + if err != nil { + return err + } + + p.lockFile, err = os.Open(podlockFile) + if err != nil { + return err + } + + err = syscall.Flock(int(p.lockFile.Fd()), syscall.LOCK_EX) + if err != nil { + return err + } + + return nil +} + +// unlock unlocks the current pod to allow it being accessed by +// other processes +func (p *Pod) unlock() error { + err := syscall.Flock(int(p.lockFile.Fd()), syscall.LOCK_UN) + if err != nil { + return err + } + + p.lockFile.Close() + + return nil +} + +func (p *Pod) createPodDirs() error { + err := os.MkdirAll(p.runPath, os.ModeDir) + if err != nil { + return err + } + + err = os.MkdirAll(p.configPath, os.ModeDir) + if err != nil { + p.storage.delete(p.id) + return err + } + + podlockFile, err := podFile(p.id, lockFileType) + if err != nil { + return err + } + + _, err = os.Stat(podlockFile) + if err != nil { + lockFile, err := os.Create(podlockFile) + if err != nil { + return err + } + lockFile.Close() + } + + return nil +} + +// createPod creates a pod from the sandbox config, the containers list, the hypervisor +// and the agent passed through the Config structure. +// It will create and store the pod structure, and then ask the hypervisor +// to physically create that pod i.e. starts a VM for that pod to eventually +// be started. +func createPod(podConfig PodConfig) (*Pod, error) { + if podConfig.valid() == false { + return nil, fmt.Errorf("Invalid pod configuration") + } + + agent, err := newAgent(podConfig.AgentType) + if err != nil { + return nil, err + } + + hypervisor, err := newHypervisor(podConfig.HypervisorType) + if err != nil { + return nil, err + } + + err = hypervisor.init(podConfig.HypervisorConfig) + if err != nil { + return nil, err + } + + p := &Pod{ + id: podConfig.ID, + hypervisor: hypervisor, + agent: agent, + storage: &filesystem{}, + config: &podConfig, + rootFs: podConfig.RootFs, + volumes: podConfig.Volumes, + containers: podConfig.Containers, + runPath: filepath.Join(runStoragePath, podConfig.ID), + configPath: filepath.Join(configStoragePath, podConfig.ID), + state: PodState{}, + } + + err = p.createPodDirs() + if err != nil { + return nil, err + } + + err = p.lock() + if err != nil { + return nil, err + } + defer p.unlock() + + err = p.hypervisor.createPod(podConfig) + if err != nil { + p.storage.delete(p.id) + return nil, err + } + + var agentConfig interface{} + + if podConfig.AgentConfig != nil { + switch podConfig.AgentConfig.(type) { + case (map[string]interface{}): + agentConfig = newAgentConfig(podConfig) + default: + agentConfig = podConfig.AgentConfig.(interface{}) + } + } else { + agentConfig = nil + } + + err = p.agent.init(agentConfig, p.hypervisor) + if err != nil { + p.storage.delete(p.id) + return nil, err + } + + state, err := p.storage.fetchState(p.id) + if err == nil && state.State != "" { + return p, nil + } + + err = p.setState(podReady) + if err != nil { + p.storage.delete(p.id) + return nil, err + } + + return p, nil +} + +// storePod stores a pod config. +func (p *Pod) storePod() error { + err := p.lock() + if err != nil { + return err + } + defer p.unlock() + + fs := filesystem{} + err = fs.storeConfig(*(p.config)) + if err != nil { + return err + } + + return nil +} + +// fetchPod fetches a pod config from a pod ID and returns a pod. +func fetchPod(podID string) (*Pod, error) { + fs := filesystem{} + config, err := fs.fetchConfig(podID) + if err != nil { + return nil, err + } + + glog.Infof("Info structure:\n%+v\n", config) + + return createPod(config) +} + +// delete deletes an already created pod. +// The VM in which the pod is running will be shut down. +func (p *Pod) delete() error { + err := p.lock() + if err != nil { + return err + } + defer p.unlock() + + state, err := p.storage.fetchState(p.id) + if err != nil { + return err + } + + if state.State != podReady { + return fmt.Errorf("Pod not %s, impossible to delete", podReady) + } + + err = p.storage.delete(p.id) + if err != nil { + return err + } + + return nil +} + +// start starts a pod. The containers that are making the pod +// will be started. +func (p *Pod) start() error { + err := p.lock() + if err != nil { + return err + } + defer p.unlock() + + state, err := p.storage.fetchState(p.id) + if err != nil { + return err + } + + err = state.validTransition(podReady, podRunning) + if err != nil { + return err + } + + podStartedCh := make(chan struct{}) + podStoppedCh := make(chan struct{}) + + go p.hypervisor.startPod(podStartedCh, podStoppedCh) + + // Wait for the pod started notification + select { + case <-podStartedCh: + break + case <-time.After(time.Second): + return fmt.Errorf("Did not receive the pod started notification") + } + + err = p.agent.start() + if err != nil { + p.stop() + return err + } + + err = p.agent.startPod(*p.config) + if err != nil { + p.stop() + return err + } + + interactive := false + for _, c := range p.config.Containers { + if c.Interactive != false && c.Console != "" { + interactive = true + break + } + } + + err = p.setState(podRunning) + if err != nil { + return err + } + + p.unlock() + + if interactive == true { + select { + case <-podStoppedCh: + err = p.setState(podReady) + if err != nil { + return err + } + + break + } + } else { + glog.Infof("Created Pod %s\n", p.ID()) + } + + return nil +} + +// stop stops a pod. The containers that are making the pod +// will be destroyed. +func (p *Pod) stop() error { + err := p.lock() + if err != nil { + return err + } + defer p.unlock() + + state, err := p.storage.fetchState(p.id) + if err != nil { + return err + } + + err = state.validTransition(podRunning, podReady) + if err != nil { + return err + } + + err = p.agent.start() + if err != nil { + return err + } + + err = p.agent.stopPod(*p.config) + if err != nil { + return err + } + + err = p.setState(podReady) + if err != nil { + return err + } + + err = p.hypervisor.stopPod() + if err != nil { + return err + } + + return nil +} + +// list lists all pod running on the host. +func (p *Pod) list() ([]Pod, error) { + return nil, nil +} + +// enter runs an executable within a pod. +func (p *Pod) enter(args []string) error { + err := p.lock() + if err != nil { + return err + } + defer p.unlock() + + return nil +} + +func (p *Pod) setState(state stateString) error { + p.state = PodState{ + State: state, + } + + err := p.storage.storeState(p.id, p.state) + if err != nil { + return err + } + + return nil +} diff --git a/qemu.go b/qemu.go new file mode 100644 index 00000000..d1e010be --- /dev/null +++ b/qemu.go @@ -0,0 +1,486 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + + ciaoQemu "github.com/01org/ciao/qemu" + "github.com/golang/glog" +) + +type qmpChannel struct { + ctx context.Context + path string + disconnectCh chan struct{} + commandCh chan qmpCommand + wg sync.WaitGroup + qmp *ciaoQemu.QMP +} + +// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. +type qemu struct { + path string + config HypervisorConfig + + hypervisorParams []string + kernelParams []string + + qmpMonitorCh qmpChannel + qmpControlCh qmpChannel + + qemuConfig ciaoQemu.Config +} + +// QMPCommand describes a VirtContainers QMP command to be sent. +type qmpCommand uint8 + +const ( + // qmpQuit will shut the VM down. + qmpQuit qmpCommand = iota + + // qmpStart will start/resume the VM. + qmpStart + + // qmpStop will stop/pause the VM. + qmpStop +) + +const defaultQemuPath = "/usr/bin/qemu-system-x86_64" + +type qmpGlogLogger struct{} + +func (l qmpGlogLogger) V(level int32) bool { + return bool(glog.V(glog.Level(level))) +} + +func (l qmpGlogLogger) Infof(format string, v ...interface{}) { + glog.InfoDepth(2, fmt.Sprintf(format, v...)) +} + +func (l qmpGlogLogger) Warningf(format string, v ...interface{}) { + glog.WarningDepth(2, fmt.Sprintf(format, v...)) +} + +func (l qmpGlogLogger) Errorf(format string, v ...interface{}) { + glog.ErrorDepth(2, fmt.Sprintf(format, v...)) +} + +var kernelDefaultParams = []Param{ + {"root", "/dev/pmem0p1"}, + {"rootflags", "dax,data=ordered,errors=remount-ro rw"}, + {"rootfstype", "ext4"}, + {"tsc", "reliable"}, + {"no_timer_check", ""}, + {"rcupdate.rcu_expedited", "1"}, + {"i8042.direct", "1"}, + {"i8042.dumbkbd", "1"}, + {"i8042.nopnp", "1"}, + {"i8042.noaux", "1"}, + {"noreplace-smp", ""}, + {"reboot", "k"}, + {"panic", "1"}, + {"console", "hvc0"}, + {"console", "hvc1"}, + {"initcall_debug", ""}, + {"init", "/usr/lib/systemd/systemd"}, + {"systemd.unit", "container.target"}, + {"iommu", "off"}, + {"quiet", ""}, + {"systemd.mask", "systemd-networkd.service"}, + {"systemd.mask", "systemd-networkd.socket"}, + {"systemd.show_status", "false"}, + {"cryptomgr.notests", ""}, +} + +func (q *qemu) buildKernelParams(config HypervisorConfig) error { + params := kernelDefaultParams + params = append(params, config.KernelParams...) + + q.kernelParams = serializeParams(params, "=") + + return nil +} + +func (q *qemu) appendVolume(devices []ciaoQemu.Device, volume Volume) []ciaoQemu.Device { + if volume.MountTag == "" || volume.HostPath == "" { + return devices + } + + devices = append(devices, + ciaoQemu.FSDevice{ + Driver: ciaoQemu.Virtio9P, + FSDriver: ciaoQemu.Local, + ID: fmt.Sprintf("extra-%s-9p", volume.MountTag), + Path: volume.HostPath, + MountTag: volume.MountTag, + SecurityModel: ciaoQemu.None, + }, + ) + + return devices +} + +func (q *qemu) appendFSDevices(devices []ciaoQemu.Device, podConfig PodConfig) []ciaoQemu.Device { + if podConfig.ID != "" { + // Add the pod rootfs + devices = append(devices, + ciaoQemu.FSDevice{ + Driver: ciaoQemu.Virtio9P, + FSDriver: ciaoQemu.Local, + ID: fmt.Sprintf("pod-%s-9p", podConfig.ID), + Path: podConfig.RootFs, + MountTag: fmt.Sprintf("pod-rootfs-%s", podConfig.ID), + SecurityModel: ciaoQemu.None, + }, + ) + } + + // Add the containers rootfs + for _, c := range podConfig.Containers { + if c.RootFs == "" || c.ID == "" { + continue + } + + devices = append(devices, + ciaoQemu.FSDevice{ + Driver: ciaoQemu.Virtio9P, + FSDriver: ciaoQemu.Local, + ID: fmt.Sprintf("ctr-%s-9p", c.ID), + Path: c.RootFs, + //MountTag: fmt.Sprintf("ctr-rootfs-%s", c.ID), + MountTag: "rootfs", // CC hack, systemd mounts rootfs. + SecurityModel: ciaoQemu.None, + }, + ) + } + + // Add the shared volumes + for _, v := range podConfig.Volumes { + devices = q.appendVolume(devices, v) + } + + return devices +} + +func (q *qemu) appendConsoles(devices []ciaoQemu.Device, podConfig PodConfig) []ciaoQemu.Device { + serial := ciaoQemu.SerialDevice{ + Driver: ciaoQemu.VirtioSerial, + ID: "serial0", + } + + devices = append(devices, serial) + + for i, c := range podConfig.Containers { + var console ciaoQemu.CharDevice + if c.Interactive == false || c.Console == "" { + consolePath := fmt.Sprintf("%s%s/console.sock", runStoragePath, podConfig.ID) + + console = ciaoQemu.CharDevice{ + Driver: ciaoQemu.Console, + Backend: ciaoQemu.Socket, + DeviceID: fmt.Sprintf("console%d", i), + ID: fmt.Sprintf("charconsole%d", i), + Path: consolePath, + } + } else { + console = ciaoQemu.CharDevice{ + Driver: ciaoQemu.Console, + Backend: ciaoQemu.Serial, + DeviceID: fmt.Sprintf("console%d", i), + ID: fmt.Sprintf("charconsole%d", i), + Path: c.Console, + } + } + + devices = append(devices, console) + } + + return devices +} + +func (q *qemu) appendImage(devices []ciaoQemu.Device, podConfig PodConfig) ([]ciaoQemu.Device, error) { + imageFile, err := os.Open(q.config.ImagePath) + if err != nil { + return nil, err + } + defer imageFile.Close() + + imageStat, err := imageFile.Stat() + if err != nil { + return nil, err + } + + object := ciaoQemu.Object{ + Driver: ciaoQemu.NVDIMM, + Type: ciaoQemu.MemoryBackendFile, + DeviceID: "nv0", + ID: "mem0", + MemPath: q.config.ImagePath, + Size: (uint64)(imageStat.Size()), + } + + devices = append(devices, object) + + return devices, nil +} + +// init intializes the Qemu structure. +func (q *qemu) init(config HypervisorConfig) error { + if config.validate() == false { + return fmt.Errorf("Invalid configuration") + } + + p := config.HypervisorPath + if p == "" { + p = defaultQemuPath + } + + q.config = config + q.path = p + + err := q.buildKernelParams(config) + if err != nil { + return err + } + + return nil +} + +func (q *qemu) qmpMonitor(connectedCh chan struct{}) { + defer func(qemu *qemu) { + if q.qmpMonitorCh.qmp != nil { + q.qmpMonitorCh.qmp.Shutdown() + } + + q.qmpMonitorCh.wg.Done() + }(q) + + cfg := ciaoQemu.QMPConfig{Logger: qmpGlogLogger{}} + qmp, ver, err := ciaoQemu.QMPStart(q.qmpMonitorCh.ctx, q.qmpMonitorCh.path, cfg, q.qmpMonitorCh.disconnectCh) + if err != nil { + glog.Errorf("Failed to connect to QEMU instance %v", err) + return + } + + q.qmpMonitorCh.qmp = qmp + + glog.Infof("QMP version %d.%d.%d", ver.Major, ver.Minor, ver.Micro) + glog.Infof("QMP capabilities %s", ver.Capabilities) + + err = q.qmpMonitorCh.qmp.ExecuteQMPCapabilities(q.qmpMonitorCh.ctx) + if err != nil { + glog.Errorf("Unable to send qmp_capabilities command: %v", err) + return + } + + close(connectedCh) + +DONE: + for { + cmd, ok := <-q.qmpMonitorCh.commandCh + if !ok { + break DONE + } + switch cmd { + case qmpStart: + err = q.qmpMonitorCh.qmp.ExecuteCont(q.qmpMonitorCh.ctx) + if err != nil { + glog.Warningf("Failed to execute start command: %v", err) + } + + case qmpStop: + err = q.qmpMonitorCh.qmp.ExecuteStop(q.qmpMonitorCh.ctx) + if err != nil { + glog.Warningf("Failed to execute stop command: %v", err) + } + + case qmpQuit: + err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) + if err != nil { + glog.Warningf("Failed to execute quit command: %v", err) + } + } + } +} + +func (q *qemu) qmpStop() error { + if q.qmpMonitorCh.commandCh == nil { + return fmt.Errorf("Invalid QMP monitor channel") + } + + q.qmpMonitorCh.commandCh <- qmpStop + + return nil +} + +func (q *qemu) qmpStart() error { + if q.qmpMonitorCh.commandCh == nil { + return fmt.Errorf("Invalid QMP monitor channel") + } + + q.qmpMonitorCh.commandCh <- qmpStart + + return nil +} + +// createPod is the Hypervisor pod creation implementation for ciaoQemu. +func (q *qemu) createPod(podConfig PodConfig) error { + var devices []ciaoQemu.Device + + machine := ciaoQemu.Machine{ + Type: "pc-lite", + Acceleration: "kvm,kernel_irqchip,nvdimm", + } + + smp := ciaoQemu.SMP{ + CPUs: 2, + Cores: 1, + Sockets: 2, + Threads: 1, + } + + memory := ciaoQemu.Memory{ + Size: "2G", + Slots: 2, + MaxMem: "3G", + } + + knobs := ciaoQemu.Knobs{ + NoUserConfig: true, + NoDefaults: true, + NoGraphic: true, + Daemonize: true, + } + + kernel := ciaoQemu.Kernel{ + Path: q.config.KernelPath, + Params: strings.Join(q.kernelParams, " "), + } + + rtc := ciaoQemu.RTC{ + Base: "utc", + DriftFix: "slew", + } + + q.qmpMonitorCh = qmpChannel{ + ctx: context.Background(), + path: fmt.Sprintf("%s/%s/%s", runStoragePath, podConfig.ID, monitorSocket), + } + + q.qmpControlCh = qmpChannel{ + ctx: context.Background(), + path: fmt.Sprintf("%s/%s/%s", runStoragePath, podConfig.ID, controlSocket), + } + + qmpSockets := []ciaoQemu.QMPSocket{ + { + Type: "unix", + Name: q.qmpMonitorCh.path, + Server: true, + NoWait: true, + }, + { + Type: "unix", + Name: q.qmpControlCh.path, + Server: true, + NoWait: true, + }, + } + + devices = q.appendFSDevices(devices, podConfig) + devices = q.appendConsoles(devices, podConfig) + devices, err := q.appendImage(devices, podConfig) + if err != nil { + return err + } + + qemuConfig := ciaoQemu.Config{ + Name: fmt.Sprintf("pod-%s", podConfig.ID), + UUID: podConfig.ID, + Path: q.path, + Ctx: q.qmpMonitorCh.ctx, + Machine: machine, + SMP: smp, + Memory: memory, + Devices: devices, + CPUModel: "host", + Kernel: kernel, + RTC: rtc, + QMPSockets: qmpSockets, + Knobs: knobs, + VGA: "none", + GlobalParam: "kvm-pit.lost_tick_policy=discard", + } + + q.qemuConfig = qemuConfig + + return nil +} + +// startPod will start the Pod's VM. +func (q *qemu) startPod(startCh, stopCh chan struct{}) error { + strErr, err := ciaoQemu.LaunchQemu(q.qemuConfig, qmpGlogLogger{}) + if err != nil { + return fmt.Errorf("%s", strErr) + } + + // Start the QMP monitoring thread + q.qmpMonitorCh.commandCh = make(chan qmpCommand) + q.qmpMonitorCh.disconnectCh = stopCh + q.qmpMonitorCh.wg.Add(1) + q.qmpMonitor(startCh) + + return nil +} + +// stopPod will stop the Pod's VM. +func (q *qemu) stopPod() error { + cfg := ciaoQemu.QMPConfig{Logger: qmpGlogLogger{}} + q.qmpControlCh.disconnectCh = make(chan struct{}) + + qmp, _, err := ciaoQemu.QMPStart(q.qmpControlCh.ctx, q.qmpControlCh.path, cfg, q.qmpControlCh.disconnectCh) + if err != nil { + glog.Errorf("Failed to connect to QEMU instance %v", err) + return err + } + + err = qmp.ExecuteQMPCapabilities(q.qmpMonitorCh.ctx) + if err != nil { + glog.Errorf("Failed to negotiate capabilities with QEMU %v", err) + return err + } + + return qmp.ExecuteSystemPowerdown(q.qmpMonitorCh.ctx) +} + +// addDevice will add extra devices to Qemu command line. +func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error { + switch devType { + case fsDev: + volume := devInfo.(Volume) + q.qemuConfig.Devices = q.appendVolume(q.qemuConfig.Devices, volume) + default: + break + } + + return nil +} diff --git a/spawner.go b/spawner.go new file mode 100644 index 00000000..84e8e311 --- /dev/null +++ b/spawner.go @@ -0,0 +1,65 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "fmt" +) + +// SpawnerType describes the type of guest agent a Pod should run. +type SpawnerType string + +const ( + // NsEnter is the nsenter spawner type + NsEnter SpawnerType = "nsenter" +) + +// Set sets an agent type based on the input string. +func (spawnerType *SpawnerType) Set(value string) error { + switch value { + case "nsenter": + *spawnerType = NsEnter + return nil + default: + return fmt.Errorf("Unknown spawner type %s", value) + } +} + +// String converts an agent type to a string. +func (spawnerType *SpawnerType) String() string { + switch *spawnerType { + case NsEnter: + return string(NsEnter) + default: + return "" + } +} + +// newSpawner returns an agent from and agent type. +func newSpawner(spawnerType SpawnerType) spawner { + switch spawnerType { + case NsEnter: + return &nsenter{} + default: + return nil + } +} + +// spawner is the virtcontainers spawner interface. +type spawner interface { + formatArgs(args []string) ([]string, error) +} diff --git a/sshd.go b/sshd.go new file mode 100644 index 00000000..fa8eee58 --- /dev/null +++ b/sshd.go @@ -0,0 +1,151 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package virtcontainers + +import ( + "fmt" + "io/ioutil" + "strings" + "time" + + "golang.org/x/crypto/ssh" +) + +// SshdConfig is a structure storing information needed for +// sshd agent initialization. +type SshdConfig struct { + Username string + PrivKeyFile string + Server string + Port string + Protocol string + + Spawner SpawnerType +} + +// sshd is an Agent interface implementation for the sshd agent. +type sshd struct { + config SshdConfig + client *ssh.Client + + spawner spawner +} + +func (c SshdConfig) validate() bool { + return true +} + +func publicKeyAuth(file string) (ssh.AuthMethod, error) { + privateBytes, err := ioutil.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("Failed to load private key") + } + + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + return nil, fmt.Errorf("Failed to parse private key") + } + + return ssh.PublicKeys(private), nil +} + +func execCmd(session *ssh.Session, cmd string) error { + stdout, err := session.CombinedOutput(cmd) + + if err != nil { + return fmt.Errorf("Failed to run %s", cmd) + } + + fmt.Printf("%s\n", stdout) + + return nil +} + +// init is the agent initialization implementation for sshd. +func (s *sshd) init(config interface{}, hypervisor hypervisor) error { + c := config.(SshdConfig) + if c.validate() == false { + return fmt.Errorf("Invalid configuration") + } + s.config = c + + s.spawner = newSpawner(c.Spawner) + + return nil +} + +// start is the agent starting implementation for sshd. +func (s *sshd) start() error { + sshAuthMethod, err := publicKeyAuth(s.config.PrivKeyFile) + if err != nil { + return err + } + sshConfig := &ssh.ClientConfig{ + User: s.config.Username, + Auth: []ssh.AuthMethod{ + sshAuthMethod, + }, + } + + for i := 0; i < 1000; i++ { + s.client, err = ssh.Dial(s.config.Protocol, s.config.Server+":"+s.config.Port, sshConfig) + if err == nil { + break + } + + select { + case <-time.After(100 * time.Millisecond): + break + } + } + + if err != nil { + return fmt.Errorf("Failed to dial: %s", err) + } + + return nil +} + +// exec is the agent command execution implementation for sshd. +func (s *sshd) exec(podID string, contID string, cmd Cmd) error { + session, err := s.client.NewSession() + if err != nil { + return fmt.Errorf("Failed to create session") + } + defer session.Close() + + if s.spawner != nil { + cmd.Args, err = s.spawner.formatArgs(cmd.Args) + if err != nil { + return err + } + } + + strCmd := strings.Join(cmd.Args, " ") + + return execCmd(session, strCmd) +} + +// startPod is the agent Pod starting implementation for sshd. +func (s *sshd) startPod(config PodConfig) error { + return nil +} + +// stopPod is the agent Pod stopping implementation for sshd. +func (s *sshd) stopPod(config PodConfig) error { + return nil +} diff --git a/virtc/main.go b/virtc/main.go new file mode 100644 index 00000000..a2bb6ae2 --- /dev/null +++ b/virtc/main.go @@ -0,0 +1,358 @@ +// +// Copyright (c) 2016 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "flag" + "fmt" + "os" + "os/user" + + "github.com/golang/glog" + "github.com/urfave/cli" + + vc "github.com/sameo/virtcontainers" +) + +var podConfigFlags = []cli.Flag{ + cli.StringFlag{ + Name: "console", + Value: "", + Usage: "the container console", + }, + + cli.StringFlag{ + Name: "bundle", + Value: "", + Usage: "the container bundle", + }, + + cli.GenericFlag{ + Name: "agent", + Value: new(vc.AgentType), + Usage: "the guest agent", + }, + + cli.GenericFlag{ + Name: "spawner", + Value: new(vc.SpawnerType), + Usage: "the guest spawner", + }, + + cli.StringFlag{ + Name: "sshd-user", + Value: "", + Usage: "the sshd user", + }, + + cli.StringFlag{ + Name: "sshd-auth-file", + Value: "", + Usage: "the sshd private key path", + }, + + cli.StringFlag{ + Name: "sshd-server", + Value: "", + Usage: "the sshd server", + }, + + cli.StringFlag{ + Name: "sshd-port", + Value: "", + Usage: "the sshd server port", + }, + + cli.StringFlag{ + Name: "hyper-ctl-sock-name", + Value: "", + Usage: "the hyperstart control socket name", + }, + + cli.StringFlag{ + Name: "hyper-tty-sock-name", + Value: "", + Usage: "the hyperstart tty socket name", + }, + + cli.StringFlag{ + Name: "hyper-ctl-sock-type", + Value: "", + Usage: "the hyperstart control socket type", + }, + + cli.StringFlag{ + Name: "hyper-tty-sock-type", + Value: "", + Usage: "the hyperstart tty socket type", + }, +} + +func buildPodConfig(context *cli.Context) (vc.PodConfig, error) { + var agConfig interface{} + + console := context.String("console") + bundle := context.String("bundle") + sshdUser := context.String("sshd-user") + sshdServer := context.String("sshd-server") + sshdPort := context.String("sshd-port") + sshdKey := context.String("sshd-auth-file") + hyperCtlSockName := context.String("hyper-ctl-sock-name") + hyperTtySockName := context.String("hyper-tty-sock-name") + hyperCtlSockType := context.String("hyper-ctl-sock-type") + hyperTtySockType := context.String("hyper-tty-sock-type") + agentType, ok := context.Generic("agent").(*vc.AgentType) + if ok != true { + return vc.PodConfig{}, fmt.Errorf("Could not convert agent type") + } + + spawnerType, ok := context.Generic("spawner").(*vc.SpawnerType) + if ok != true { + return vc.PodConfig{}, fmt.Errorf("Could not convert spawner type") + } + + u, _ := user.Current() + if sshdUser == "" { + sshdUser = u.Username + } + + interactive := false + if console != "" { + interactive = true + } + + cmd := vc.Cmd{ + Args: []string{"/bin/bash", "echo", "hello"}, + WorkDir: "/", + } + + container := vc.ContainerConfig{ + ID: "1", + RootFs: bundle, + Interactive: interactive, + Console: console, + Cmd: cmd, + } + + containers := []vc.ContainerConfig{ + container, + } + + hypervisorConfig := vc.HypervisorConfig{ + KernelPath: "/usr/share/clear-containers/vmlinux.container", + ImagePath: "/usr/share/clear-containers/clear-containers.img", + HypervisorPath: "/usr/bin/qemu-lite-system-x86_64", + } + + switch *agentType { + case vc.SSHdAgent: + agConfig = vc.SshdConfig{ + Username: sshdUser, + PrivKeyFile: sshdKey, + Server: sshdServer, + Port: sshdPort, + Protocol: "tcp", + Spawner: *spawnerType, + } + case vc.HyperstartAgent: + agConfig = vc.HyperConfig{ + SockCtlName: hyperCtlSockName, + SockTtyName: hyperTtySockName, + SockCtlType: hyperCtlSockType, + SockTtyType: hyperTtySockType, + } + default: + agConfig = nil + } + + podConfig := vc.PodConfig{ + HypervisorType: vc.QemuHypervisor, + HypervisorConfig: hypervisorConfig, + + AgentType: *agentType, + AgentConfig: agConfig, + + Containers: containers, + } + + return podConfig, nil +} + +func runPod(context *cli.Context) error { + podConfig, err := buildPodConfig(context) + if err != nil { + return fmt.Errorf("Could not build pod config: %s\n", err) + } + + _, err = vc.RunPod(podConfig) + if err != nil { + return fmt.Errorf("Could not run pod: %s\n", err) + } + + return nil +} + +func createPod(context *cli.Context) error { + podConfig, err := buildPodConfig(context) + if err != nil { + return fmt.Errorf("Could not build pod config: %s\n", err) + } + + p, err := vc.CreatePod(podConfig) + if err != nil { + return fmt.Errorf("Could not create pod: %s\n", err) + } + + fmt.Printf("Created pod %s\n", p.ID()) + + return nil +} + +func deletePod(context *cli.Context) error { + _, err := vc.DeletePod(context.String("id")) + if err != nil { + return fmt.Errorf("Could not delete pod: %s\n", err) + } + + return nil +} + +func startPod(context *cli.Context) error { + _, err := vc.StartPod(context.String("id")) + if err != nil { + return fmt.Errorf("Could not delete pod: %s\n", err) + } + + return nil +} + +func stopPod(context *cli.Context) error { + _, err := vc.StopPod(context.String("id")) + if err != nil { + return fmt.Errorf("Could not stop pod: %s\n", err) + } + + return nil +} + +func listPods(context *cli.Context) error { + err := vc.ListPod() + if err != nil { + return fmt.Errorf("Could not list pod: %s\n", err) + } + + return nil +} + +var runPodCommand = cli.Command{ + Name: "run", + Usage: "run a pod", + Flags: podConfigFlags, + Action: func(context *cli.Context) error { + return runPod(context) + }, +} + +var createPodCommand = cli.Command{ + Name: "create", + Usage: "create a pod", + Flags: podConfigFlags, + Action: func(context *cli.Context) error { + return createPod(context) + }, +} + +var deletePodCommand = cli.Command{ + Name: "delete", + Usage: "delete an existing pod", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Value: "", + Usage: "the pod identifier", + }, + }, + Action: func(context *cli.Context) error { + return deletePod(context) + }, +} + +var startPodCommand = cli.Command{ + Name: "start", + Usage: "start an existing pod", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Value: "", + Usage: "the pod identifier", + }, + }, + Action: func(context *cli.Context) error { + return startPod(context) + }, +} + +var stopPodCommand = cli.Command{ + Name: "stop", + Usage: "stop an existing pod", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Value: "", + Usage: "the pod identifier", + }, + }, + Action: func(context *cli.Context) error { + return stopPod(context) + }, +} + +var listPodsCommand = cli.Command{ + Name: "list", + Usage: "list all existing pods", + Action: func(context *cli.Context) error { + return listPods(context) + }, +} + +func main() { + flag.Parse() + + virtc := cli.NewApp() + virtc.Name = "VirtContainers CLI" + virtc.Version = "0.0.1" + + virtc.Commands = []cli.Command{ + { + Name: "pod", + Usage: "pod commands", + Subcommands: []cli.Command{ + createPodCommand, + deletePodCommand, + listPodsCommand, + runPodCommand, + startPodCommand, + stopPodCommand, + }, + }, + } + + err := virtc.Run(os.Args) + if err != nil { + glog.Fatal(err) + } +}