Skip to content

Commit 039c267

Browse files
dcantahdcantah
authored
Merge pull request microsoft#393 from dcantah/removenetnscfg
Remove netnscfg gcstools utility
2 parents 4db65a8 + 80883ee commit 039c267

File tree

5 files changed

+246
-267
lines changed

5 files changed

+246
-267
lines changed

‎Makefile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ SRCROOT=$(dir $(abspath $(firstword $(MAKEFILE_LIST))))
1818

1919
# The link aliases for gcstools
2020
GCS_TOOLS=\
21-
netnscfg \
2221
nvidiaPrestartHook
2322

2423
.PHONY: all always rootfs test

‎internal/network/netns.go‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
package network
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net"
7+
"os/exec"
8+
"runtime"
9+
"time"
10+
11+
"github.com/Microsoft/opengcs/internal/log"
12+
"github.com/Microsoft/opengcs/service/gcs/prot"
13+
"github.com/pkg/errors"
14+
"github.com/vishvananda/netlink"
15+
"github.com/vishvananda/netns"
16+
)
17+
18+
// MoveInterfaceToNS moves the adapter with interface name `ifStr` to the network namespace
19+
// of `pid`.
20+
func MoveInterfaceToNS(ifStr string, pid int) error {
21+
// Get a reference to the interface and make sure it's down
22+
link, err := netlink.LinkByName(ifStr)
23+
if err != nil {
24+
return errors.Wrapf(err, "netlink.LinkByName(%s) failed", ifStr)
25+
}
26+
if err := netlink.LinkSetDown(link); err != nil {
27+
return errors.Wrapf(err, "netlink.LinkSetDown(%#v) failed", link)
28+
}
29+
30+
// Move the interface to the new network namespace
31+
if err := netlink.LinkSetNsPid(link, pid); err != nil {
32+
return errors.Wrapf(err, "netlink.SetNsPid(%#v, %d) failed", link, pid)
33+
}
34+
return nil
35+
}
36+
37+
// DoInNetNS is a utility to run a function `run` inside of a specific network namespace
38+
// `ns`. This is accomplished by locking the current goroutines thread to prevent the goroutine
39+
// from being scheduled to a new thread during execution of `run`. The threads original network namespace
40+
// will be rejoined on exit.
41+
func DoInNetNS(ns netns.NsHandle, run func() error) error {
42+
runtime.LockOSThread()
43+
defer runtime.UnlockOSThread()
44+
45+
origNs, err := netns.Get()
46+
if err != nil {
47+
return errors.Wrap(err, "failed to get current network namespace")
48+
}
49+
defer origNs.Close()
50+
51+
if err := netns.Set(ns); err != nil {
52+
return errors.Wrapf(err, "failed to set network namespace to %v", ns)
53+
}
54+
// Defer so we can re-enter the threads original netns on exit.
55+
defer netns.Set(origNs)
56+
57+
return run()
58+
}
59+
60+
// NetNSConfig moves a network interface into a network namespace and
61+
// configures it.
62+
//
63+
// This function MUST be used in tandem with `DoInNetNS` or some other means that ensures that the goroutine
64+
// executing this code stays on the same thread.
65+
func NetNSConfig(ctx context.Context, ifStr string, nsPid int, adapter *prot.NetworkAdapter) error {
66+
if ifStr == "" || nsPid == -1 || adapter == nil {
67+
return errors.New("All three arguments must be specified")
68+
}
69+
70+
if adapter.NatEnabled {
71+
log.G(ctx).Debugf("Configure %s in %d with: %s/%d gw=%s", ifStr, nsPid, adapter.AllocatedIPAddress, adapter.HostIPPrefixLength, adapter.HostIPAddress)
72+
} else {
73+
log.G(ctx).Debugf("Configure %s in %d with DHCP", ifStr, nsPid)
74+
}
75+
76+
log.G(ctx).Debug("Obtaining current namespace")
77+
ns, err := netns.Get()
78+
if err != nil {
79+
return errors.Wrap(err, "netns.Get() failed")
80+
}
81+
defer ns.Close()
82+
83+
log.G(ctx).Debugf("New network namespace from PID %d is %v", nsPid, ns)
84+
85+
// Re-Get a reference to the interface (it may be a different ID in the new namespace)
86+
log.G(ctx).Debug("Getting reference to interface")
87+
link, err := netlink.LinkByName(ifStr)
88+
if err != nil {
89+
return errors.Wrapf(err, "netlink.LinkByName(%s) failed", ifStr)
90+
}
91+
92+
// User requested non-default MTU size
93+
if adapter.EncapOverhead != 0 {
94+
log.G(ctx).Debug("EncapOverhead non-zero, will set MTU")
95+
mtu := link.Attrs().MTU - int(adapter.EncapOverhead)
96+
log.G(ctx).Debugf("mtu %d", mtu)
97+
if err = netlink.LinkSetMTU(link, mtu); err != nil {
98+
return errors.Wrapf(err, "netlink.LinkSetMTU(%#v, %d) failed", link, mtu)
99+
}
100+
}
101+
102+
// Configure the interface
103+
if adapter.NatEnabled {
104+
log.G(ctx).Debug("Nat enabled - configuring interface")
105+
metric := 1
106+
if adapter.EnableLowMetric {
107+
metric = 500
108+
}
109+
110+
// Bring the interface up
111+
if err := netlink.LinkSetUp(link); err != nil {
112+
return errors.Wrapf(err, "netlink.LinkSetUp(%#v) failed", link)
113+
}
114+
// Set IP address
115+
addr := &net.IPNet{
116+
IP: net.ParseIP(adapter.AllocatedIPAddress),
117+
// TODO(rn): This assumes/hardcodes IPv4
118+
Mask: net.CIDRMask(int(adapter.HostIPPrefixLength), 32)}
119+
ipAddr := &netlink.Addr{IPNet: addr, Label: ""}
120+
if err := netlink.AddrAdd(link, ipAddr); err != nil {
121+
return errors.Wrapf(err, "netlink.AddrAdd(%#v, %#v) failed", link, ipAddr)
122+
}
123+
// Set gateway
124+
if adapter.HostIPAddress != "" {
125+
gw := net.ParseIP(adapter.HostIPAddress)
126+
127+
if !addr.Contains(gw) {
128+
// In the case that a gw is not part of the subnet we are setting gw for,
129+
// a new addr containing this gw address need to be added into the link to avoid getting
130+
// unreachable error when adding this out-of-subnet gw route
131+
log.G(ctx).Debugf("gw is outside of the subnet: Configure %s in %d with: %s/%d gw=%s\n",
132+
ifStr, nsPid, adapter.AllocatedIPAddress, adapter.HostIPPrefixLength, adapter.HostIPAddress)
133+
addr2 := &net.IPNet{
134+
IP: net.ParseIP(adapter.HostIPAddress),
135+
Mask: net.CIDRMask(32, 32)} // This assumes/hardcodes IPv4
136+
ipAddr2 := &netlink.Addr{IPNet: addr2, Label: ""}
137+
if err := netlink.AddrAdd(link, ipAddr2); err != nil {
138+
return errors.Wrapf(err, "netlink.AddrAdd(%#v, %#v) failed", link, ipAddr2)
139+
}
140+
}
141+
142+
if !adapter.EnableLowMetric {
143+
route := netlink.Route{
144+
Scope: netlink.SCOPE_UNIVERSE,
145+
LinkIndex: link.Attrs().Index,
146+
Gw: gw,
147+
Priority: metric, // This is what ip route add does
148+
}
149+
if err := netlink.RouteAdd(&route); err != nil {
150+
return errors.Wrapf(err, "netlink.RouteAdd(%#v) failed", route)
151+
}
152+
} else {
153+
// add a route rule for the new interface so packets coming on this interface
154+
// always go out the same interface
155+
srcNet := &net.IPNet{IP: net.ParseIP(adapter.AllocatedIPAddress), Mask: net.CIDRMask(32, 32)}
156+
rule := netlink.NewRule()
157+
rule.Table = 101
158+
rule.Src = srcNet
159+
rule.Priority = 5
160+
161+
if err := netlink.RuleAdd(rule); err != nil {
162+
return errors.Wrapf(err, "netlink.RuleAdd(%#v) failed", rule)
163+
}
164+
165+
// add the default route in that interface specific table
166+
route := netlink.Route{
167+
Scope: netlink.SCOPE_UNIVERSE,
168+
LinkIndex: link.Attrs().Index,
169+
Gw: gw,
170+
Table: rule.Table,
171+
Priority: metric,
172+
}
173+
if err := netlink.RouteAdd(&route); err != nil {
174+
return errors.Wrapf(err, "netlink.RouteAdd(%#v) failed", route)
175+
}
176+
177+
}
178+
}
179+
} else {
180+
log.G(ctx).Debug("Execing udhcpc with timeout...")
181+
cmd := exec.Command("udhcpc", "-q", "-i", ifStr, "-s", "/sbin/udhcpc_config.script")
182+
183+
done := make(chan error)
184+
go func() {
185+
done <- cmd.Wait()
186+
}()
187+
defer close(done)
188+
189+
select {
190+
case <-time.After(30 * time.Second):
191+
var cos string
192+
co, err := cmd.CombinedOutput() // In case it has written something
193+
if err != nil {
194+
cos = string(co)
195+
}
196+
cmd.Process.Kill()
197+
log.G(ctx).Debugf("udhcpc timed out [%s]", cos)
198+
return fmt.Errorf("udhcpc timed out. Failed to get DHCP address: %s", cos)
199+
case err := <-done:
200+
var cos string
201+
co, err := cmd.CombinedOutput() // Something should be on stderr
202+
if err != nil {
203+
cos = string(co)
204+
}
205+
if err != nil {
206+
log.G(ctx).WithError(err).Debugf("udhcpc failed [%s]", cos)
207+
return errors.Wrapf(err, "process failed (%s)", cos)
208+
}
209+
}
210+
var cos string
211+
co, err := cmd.CombinedOutput()
212+
if err != nil {
213+
cos = string(co)
214+
}
215+
log.G(ctx).Debugf("udhcpc succeeded: %s", cos)
216+
}
217+
218+
// Add some debug logging
219+
curNS, _ := netns.Get()
220+
// Refresh link attributes/state
221+
link, _ = netlink.LinkByIndex(link.Attrs().Index)
222+
attr := link.Attrs()
223+
addrs, _ := netlink.AddrList(link, 0)
224+
225+
log.G(ctx).Debugf("%v: %s[idx=%d,type=%s] is %v", curNS, attr.Name, attr.Index, link.Type(), attr.OperState)
226+
for _, addr := range addrs {
227+
log.G(ctx).Debugf(" %v", addr)
228+
}
229+
230+
return nil
231+
}

‎internal/runtime/hcsv2/network.go‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,7 @@ package hcsv2
22

33
import (
44
"context"
5-
"encoding/json"
65
"fmt"
7-
"os/exec"
8-
"strconv"
96
"strings"
107
"sync"
118
"time"
@@ -15,6 +12,7 @@ import (
1512
"github.com/Microsoft/opengcs/service/gcs/gcserr"
1613
"github.com/Microsoft/opengcs/service/gcs/prot"
1714
"github.com/pkg/errors"
15+
"github.com/vishvananda/netns"
1816
"go.opencensus.io/trace"
1917
)
2018

@@ -249,8 +247,6 @@ func (nin *nicInNamespace) assignToPid(ctx context.Context, pid int) (err error)
249247
trace.StringAttribute("ifname", nin.ifname),
250248
trace.Int64Attribute("pid", int64(pid)))
251249

252-
// TODO: netnscfg is not coded for v2 but since they are almost the same
253-
// just convert the parts of the adapter here.
254250
v1Adapter := &prot.NetworkAdapter{
255251
NatEnabled: nin.adapter.IPAddress != "",
256252
AllocatedIPAddress: nin.adapter.IPAddress,
@@ -260,17 +256,23 @@ func (nin *nicInNamespace) assignToPid(ctx context.Context, pid int) (err error)
260256
EncapOverhead: nin.adapter.EncapOverhead,
261257
}
262258

263-
cfg, err := json.Marshal(v1Adapter)
264-
if err != nil {
265-
return errors.Wrap(err, "failed to marshal adapter struct to JSON")
259+
if err := network.MoveInterfaceToNS(nin.ifname, pid); err != nil {
260+
return errors.Wrapf(err, "failed to move interface %s to network namespace", nin.ifname)
266261
}
267262

268-
out, err := exec.Command("netnscfg",
269-
"-if", nin.ifname,
270-
"-nspid", strconv.Itoa(pid),
271-
"-cfg", string(cfg)).CombinedOutput()
263+
// Get a reference to the new network namespace
264+
ns, err := netns.GetFromPid(pid)
272265
if err != nil {
273-
return errors.Wrapf(err, "failed to configure adapter aid: %s, if id: %s %s", nin.adapter.ID, nin.ifname, string(out))
266+
return errors.Wrapf(err, "netns.GetFromPid(%d) failed", pid)
267+
}
268+
defer ns.Close()
269+
270+
netNSCfg := func() error {
271+
return network.NetNSConfig(ctx, nin.ifname, pid, v1Adapter)
272+
}
273+
274+
if err := network.DoInNetNS(ns, netNSCfg); err != nil {
275+
return errors.Wrapf(err, "failed to configure adapter aid: %s, if id: %s", nin.adapter.ID, nin.ifname)
274276
}
275277
nin.assignedPid = pid
276278
return nil

‎service/gcsutils/gcstools/main.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
)
88

99
var commands = map[string]func(){
10-
"netnscfg": netnsConfigMain,
1110
"nvidiaPrestartHook": nvidiaPrestartHookMain,
1211
}
1312

0 commit comments

Comments
 (0)