Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

fix daemon restore #511

Merged
merged 7 commits into from
Feb 14, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,8 +259,9 @@ func (daemon *Daemon) initDefaultLog(c *apitypes.HyperConfig) {
}

func (daemon *Daemon) GetPodNum() int64 {
pods, err := daemon.db.ListPod()
pods, err := pod.ListAllPods(daemon.db)
if err != nil {
glog.Errorf("fail to list pods: %v", err)
return 0
}
return int64(len(pods))
Expand Down
8 changes: 6 additions & 2 deletions daemon/daemondb/daemondb.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ func (d *DaemonDB) Close() error {
return d.db.Close()
}

func (d *DaemonDB) Delete(key []byte) error {
return d.db.Delete(key, nil)
}

func (d *DaemonDB) Get(key []byte) ([]byte, error) {
data, err := d.db.Get(key, nil)
if err != nil {
Expand Down Expand Up @@ -227,8 +231,8 @@ func (d *DaemonDB) PrefixList2Chan(prefix []byte, keyFilter KeyFilter) chan *KVP
for iter.Next() {
glog.V(3).Infof("got key from leveldb %s", string(iter.Key()))
if keyFilter == nil || keyFilter(iter.Key()) {
k := make([]byte, 0, len(iter.Key()))
v := make([]byte, 0, len(iter.Value()))
k := make([]byte, len(iter.Key()))
v := make([]byte, len(iter.Value()))
copy(k, iter.Key())
copy(v, iter.Value())
ch <- &KVPair{k, v}
Expand Down
2 changes: 1 addition & 1 deletion daemon/pod/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ func (c *Container) loadJson() *dockertypes.ContainerJSON {
}

n := strings.TrimLeft(rsp.Name, "/")
if c.spec.Name != rsp.Name {
if c.spec.Name != n {
c.Log(ERROR, "name mismatch of loaded container, loaded is %s", n)
c.spec.Id = ""
return nil
Expand Down
24 changes: 20 additions & 4 deletions daemon/pod/decommission.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,14 @@ func (p *XPod) Remove(force bool) error {

os.RemoveAll(path.Join(utils.HYPER_ROOT, "hosts", p.Id()))

//TODO get created volumes and remove them
for id, c := range p.containers {
p.factory.registry.ReleaseContainer(id, c.SpecName())
p.factory.engine.ContainerRm(id, &dockertypes.ContainerRmConfig{false, false, false})
}
//TODO should remove items in daemondb: daemon.db.DeletePod(p.Id)

//remove pod(including all containers/volumes/interfaces) in daemondb
p.removeFromDB()

if p.DelayDeleteOn() {
p.Log(DEBUG, "should wait periodical clean up")
p.factory.registry.Release(p.Id())
Expand Down Expand Up @@ -295,8 +297,22 @@ func (p *XPod) RemoveContainer(id string) error {
}
}

//TODO: remove volumes those created during container creating
//TODO: remove containers in daemondb daemon.db.DeleteP2C(p.Id)
//remove volumes from daemondb
for _, vName := range removedVols {
if v, ok := p.volumes[vName]; ok {
if err = v.removeFromDB(); err != nil {
return err
}
}
}
// remove container in daemondb.
if err = c.removeFromDB(); err != nil {
return err
}
// update layout to remove container from pod layout.
if err = p.saveLayout(); err != nil {
return err
}

return nil
}
Expand Down
2 changes: 1 addition & 1 deletion daemon/pod/networks.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (inf *Interface) prepare() error {
return err
}
inf.descript = &runv.InterfaceDescription{
Id: setting.IPAddress,
Id: inf.spec.Ifname,
Lo: false,
Bridge: setting.Bridge,
Ip: setting.IPAddress,
Expand Down
121 changes: 119 additions & 2 deletions daemon/pod/persist.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ const (
IF_KEY_FMT = "IF-%s-%s"
)

func ListAllPods(db *daemondb.DaemonDB) ([][]byte, error) {
return db.PrefixListKey([]byte(LAYOUT_KEY_PREFIX), nil)
}

func LoadAllPods(db *daemondb.DaemonDB) chan *types.PersistPodLayout {
kvchan := db.PrefixList2Chan([]byte(LAYOUT_KEY_PREFIX), nil)
if kvchan == nil {
Expand Down Expand Up @@ -132,6 +136,10 @@ func (p *XPod) savePod() error {
return err
}

if err := p.saveSandbox(); err != nil {
return err
}

if err := p.savePodMeta(); err != nil {
return err
}
Expand Down Expand Up @@ -166,19 +174,90 @@ func (p *XPod) savePod() error {
return saveMessage(p.factory.db, fmt.Sprintf(LAYOUT_KEY_FMT, p.Id()), pl, p, "pod layout")
}

func (p *XPod) removeFromDB() (err error) {
// remove pod layout entry first.
if err = removeMessage(p.factory.db, fmt.Sprintf(LAYOUT_KEY_FMT, p.Id()), p, "pod layout"); err != nil {
return err
}

for _, i := range p.interfaces {
if err = i.removeFromDB(); err != nil {
return err
}
}

for _, v := range p.volumes {
if err = v.removeFromDB(); err != nil {
return err
}
}

for _, c := range p.containers {
if err = c.removeFromDB(); err != nil {
return err
}
}

if err = p.removePodMetaFromDB(); err != nil {
return err
}

if err = p.removeSandboxFromDB(); err != nil {
return err
}

if err = p.removeGlobalSpecFromDB(); err != nil {
return err
}

return nil
}

func (p *XPod) saveLayout() error {
var (
containers = make([]string, 0, len(p.containers))
volumes = make([]string, 0, len(p.volumes))
interfaces = make([]string, 0, len(p.interfaces))
)

for cid := range p.containers {
containers = append(containers, cid)
}

for vid := range p.volumes {
volumes = append(volumes, vid)
}

for inf := range p.interfaces {
interfaces = append(interfaces, inf)
}

pl := &types.PersistPodLayout{
Id: p.Id(),
Containers: containers,
Volumes: volumes,
Interfaces: interfaces,
}
return saveMessage(p.factory.db, fmt.Sprintf(LAYOUT_KEY_FMT, p.Id()), pl, p, "pod layout")
}

func (p *XPod) saveGlobalSpec() error {
return saveMessage(p.factory.db, fmt.Sprintf(PS_KEY_FMT, p.Id()), p.globalSpec, p, "global spec")
}

func loadGloabalSpec(db *daemondb.DaemonDB, id string) (*types.UserPod, error) {
var spec types.UserPod
err := loadMessage(db, fmt.Sprintf(LAYOUT_KEY_FMT, id), &spec, nil, fmt.Sprintf("spec for %s", id))
err := loadMessage(db, fmt.Sprintf(PS_KEY_FMT, id), &spec, nil, fmt.Sprintf("spec for %s", id))
if err != nil {
return nil, err
}
return &spec, nil
}

func (p *XPod) removeGlobalSpecFromDB() error {
return removeMessage(p.factory.db, fmt.Sprintf(PS_KEY_FMT, p.Id()), p, "global spec")
}

func (p *XPod) savePodMeta() error {
meta := &types.PersistPodMeta{
Id: p.Id(),
Expand Down Expand Up @@ -206,6 +285,10 @@ func (p *XPod) loadPodMeta() error {
return nil
}

func (p *XPod) removePodMetaFromDB() error {
return removeMessage(p.factory.db, fmt.Sprintf(PM_KEY_FMT, p.Id()), p, "pod meta")
}

func (c *Container) saveContainer() error {
cx := &types.PersistContainer{
Id: c.Id(),
Expand Down Expand Up @@ -236,6 +319,10 @@ func (p *XPod) loadContainer(id string) error {
return nil
}

func (c *Container) removeFromDB() error {
return removeMessage(c.p.factory.db, fmt.Sprintf(CX_KEY_FMT, c.Id()), c, "container info")
}

func (v *Volume) saveVolume() error {
vx := &types.PersistVolume{
Name: v.spec.Name,
Expand All @@ -259,6 +346,10 @@ func (p *XPod) loadVolume(id string) error {
return nil
}

func (v *Volume) removeFromDB() error {
return removeMessage(v.p.factory.db, fmt.Sprintf(VX_KEY_FMT, v.p.Id(), v.spec.Name), v, "volume info")
}

func (inf *Interface) saveInterface() error {
ix := &types.PersistInterface{
Id: inf.descript.Id,
Expand All @@ -281,15 +372,31 @@ func (p *XPod) loadInterface(id string) error {
return nil
}

func (inf *Interface) removeFromDB() error {
return removeMessage(inf.p.factory.db, fmt.Sprintf(IF_KEY_FMT, inf.p.Id(), inf.descript.Id), inf, "interface info")
}

func (p *XPod) saveSandbox() error {
var sb types.SandboxPersistInfo
if p.sandbox != nil {
// TODO: dump sandbox info from runv VM
}
return saveMessage(p.factory.db, fmt.Sprintf(SB_KEY_FMT, p.Id()), &sb, p, "sandbox info")
}

func (p *XPod) loadSandbox() error {
var sb types.SandboxPersistInfo
err := loadMessage(p.factory.db, fmt.Sprintf(SB_KEY_FMT, p.Id()), &sb, p, "load sandbox info")
err := loadMessage(p.factory.db, fmt.Sprintf(SB_KEY_FMT, p.Id()), &sb, p, "sandbox info")
if err != nil {
return err
}
return p.reconnectSandbox(sb.Id, sb.PersistInfo)
}

func (p *XPod) removeSandboxFromDB() error {
return removeMessage(p.factory.db, fmt.Sprintf(SB_KEY_FMT, p.Id()), p, "sandbox info")
}

func saveMessage(db *daemondb.DaemonDB, key string, message proto.Message, owner hlog.LogOwner, op string) error {
pm, err := proto.Marshal(message)
if err != nil {
Expand Down Expand Up @@ -318,3 +425,13 @@ func loadMessage(db *daemondb.DaemonDB, key string, message proto.Message, owner
}
return nil
}

func removeMessage(db *daemondb.DaemonDB, key string, owner hlog.LogOwner, op string) error {
err := db.Delete([]byte(key))
if err != nil {
hlog.HLog(ERROR, owner, 2, "failed to remove %s: %v", op, err)
return err
}
hlog.HLog(DEBUG, owner, 2, "%s removed from db", op)
return nil
}
9 changes: 5 additions & 4 deletions daemon/pod/provision.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,16 @@ func CreateXPod(factory *PodFactory, spec *apitypes.UserPod) (*XPod, error) {

p.initPodInfo()

//TODO: write the daemon db
//daemon.WritePodAndContainers(pod.Id)

// reserve again in case container is created
err = p.reserveNames(spec.Containers)
if err != nil {
return nil, err
}

if err = p.savePod(); err != nil {
return nil, err
}

return p, nil
}

Expand Down Expand Up @@ -250,7 +251,7 @@ func (p *XPod) reconnectSandbox(sandboxId string, pinfo []byte) error {
)

if sandboxId != "" {
sandbox, err = associateSandbox(sandboxId)
sandbox, err = associateSandbox(sandboxId, pinfo)
if err != nil {
p.Log(ERROR, err)
sandbox = nil
Expand Down
19 changes: 7 additions & 12 deletions daemon/pod/sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,13 @@ func startSandbox(f factory.Factory, cpu, mem int, kernel, initrd string) (vm *h
return vm, err
}

func associateSandbox(id string) (vm *hypervisor.Vm, err error) {
//vmData, err := daemon.db.GetVM(id)
//if err != nil {
// return err
//}
//vm := hypervisor.NewVm(id, 0, 0, false)
//err = vm.AssociateVm(vmData)
//if err != nil {
// return nil, err
//}
//return vm, nil
return nil, nil
func associateSandbox(id string, vmData []byte) (vm *hypervisor.Vm, err error) {
vm = hypervisor.NewVm(id, 0, 0, false)
err = vm.AssociateVm(vmData)
if err != nil {
return nil, err
}
return vm, nil
}

func dissociateSandbox(sandbox *hypervisor.Vm, retry int) error {
Expand Down