Skip to content

Commit a8b3199

Browse files
pleshakovpleshakov
authored
Add configuration file and runtime management of NGINX (#77)
Problem: (1) Add/update/delete NGINX configuration files to/from the file system. (2) Reload NGINX to apply changes in the configuration files. Solution: For (1): Add a new component - file.Manager - that allows writing and removing configuration files for a server. The Manager writes server configuration files to /etc/nginx/conf.d folder. This folder is available to both gateway and NGINX containers as part of the shared volume /etc/nginx For (2): Add a new component - runtime.Manager - that allows reload NGINX. The Manager reloads NGINX by sending a HUP signal to the NGINX master process. Because the gateway process is running under user 1001 (rather than root), during the image build we add the CAP_KILL capability to the gateway binary.
1 parent 72ad354 commit a8b3199

File tree

11 files changed

+681
-32
lines changed

11 files changed

+681
-32
lines changed

‎build/Dockerfile

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,25 @@ RUN go mod download
1111
COPY cmd /go/src/github.com/nginxinc/nginx-gateway-kubernetes/cmd
1212
COPY internal /go/src/github.com/nginxinc/nginx-gateway-kubernetes/internal
1313
COPY pkg /go/src/github.com/nginxinc/nginx-gateway-kubernetes/pkg
14-
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -a -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE}" -o ./build/.out/gateway .
14+
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -a -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE}" -o gateway .
15+
16+
FROM alpine:3.15 as capabilizer
17+
RUN apk add --no-cache libcap
18+
19+
FROM capabilizer as local-capabilizer
20+
COPY ./build/.out/gateway /usr/bin/
21+
RUN setcap 'cap_kill=+ep' /usr/bin/gateway
22+
23+
FROM capabilizer as container-capabilizer
24+
COPY --from=builder /go/src/github.com/nginxinc/nginx-gateway-kubernetes/cmd/gateway/gateway /usr/bin/
25+
RUN setcap 'cap_kill=+ep' /usr/bin/gateway
1526

1627
FROM scratch as common
1728
USER 1001:1001
1829
ENTRYPOINT [ "/usr/bin/gateway" ]
1930

2031
FROM common as container
21-
COPY --from=builder /go/src/github.com/nginxinc/nginx-gateway-kubernetes/cmd/gateway/gateway /usr/bin/
32+
COPY --from=container-capabilizer /usr/bin/gateway /usr/bin/
2233

2334
FROM common as local
24-
COPY ./build/.out/gateway /usr/bin/
35+
COPY --from=local-capabilizer /usr/bin/gateway /usr/bin/

‎deploy/manifests/nginx-gateway.yaml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,21 +72,30 @@ spec:
7272
labels:
7373
app: nginx-gateway
7474
spec:
75+
shareProcessNamespace: true
7576
serviceAccountName: nginx-gateway
7677
volumes:
7778
- name: nginx-config
7879
emptyDir: { }
7980
initContainers:
80-
- image: busybox:1.34
81+
- image: busybox:1.34 # FIXME(pleshakov): use gateway container to init the Config with proper main config
8182
name: nginx-config-initializer
82-
command: [ 'sh', '-c', 'echo "events {} http { server { default_type text/plain; return 200 \"hello from \$hostname\n\"; } }" > /etc/nginx/nginx.conf' ]
83+
command: [ 'sh', '-c', 'echo "events {} pid /etc/nginx/nginx.pid; http { include /etc/nginx/conf.d/*.conf; server { default_type text/html; return 404; } }" > /etc/nginx/nginx.conf && mkdir /etc/nginx/conf.d && chown 1001:0 /etc/nginx/conf.d' ]
8384
volumeMounts:
8485
- name: nginx-config
8586
mountPath: /etc/nginx
8687
containers:
8788
- image: nginx-gateway:0.0.1
8889
imagePullPolicy: IfNotPresent
8990
name: nginx-gateway
91+
volumeMounts:
92+
- name: nginx-config
93+
mountPath: /etc/nginx
94+
securityContext:
95+
runAsUser: 1001
96+
# FIXME(pleshakov) - figure out which capabilities are required
97+
# dropping ALL and adding only CAP_KILL doesn't work
98+
# Note: CAP_KILL is needed for sending HUP signal to NGINX main process
9099
args:
91100
- --gateway-ctlr-name=k8s-gateway.nginx.org/nginx-gateway/gateway
92101
- image: nginx:1.21.3

‎internal/events/loop.go

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import (
66

77
"github.com/go-logr/logr"
88
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/config"
9+
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/file"
10+
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/runtime"
911
"github.com/nginxinc/nginx-gateway-kubernetes/internal/state"
1012
"github.com/nginxinc/nginx-gateway-kubernetes/internal/status"
1113
apiv1 "k8s.io/api/core/v1"
@@ -14,12 +16,14 @@ import (
1416

1517
// EventLoop is the main event loop of the Gateway.
1618
type EventLoop struct {
17-
conf state.Configuration
18-
serviceStore state.ServiceStore
19-
generator config.Generator
20-
eventCh <-chan interface{}
21-
logger logr.Logger
22-
statusUpdater status.Updater
19+
conf state.Configuration
20+
serviceStore state.ServiceStore
21+
generator config.Generator
22+
eventCh <-chan interface{}
23+
logger logr.Logger
24+
statusUpdater status.Updater
25+
nginxFileMgr file.Manager
26+
nginxRuntimeMgr runtime.Manager
2327
}
2428

2529
// NewEventLoop creates a new EventLoop.
@@ -30,14 +34,18 @@ func NewEventLoop(
3034
eventCh <-chan interface{},
3135
statusUpdater status.Updater,
3236
logger logr.Logger,
37+
nginxFileMgr file.Manager,
38+
nginxRuntimeMgr runtime.Manager,
3339
) *EventLoop {
3440
return &EventLoop{
35-
conf: conf,
36-
serviceStore: serviceStore,
37-
generator: generator,
38-
eventCh: eventCh,
39-
statusUpdater: statusUpdater,
40-
logger: logger.WithName("eventLoop"),
41+
conf: conf,
42+
serviceStore: serviceStore,
43+
generator: generator,
44+
eventCh: eventCh,
45+
statusUpdater: statusUpdater,
46+
logger: logger.WithName("eventLoop"),
47+
nginxFileMgr: nginxFileMgr,
48+
nginxRuntimeMgr: nginxRuntimeMgr,
4149
}
4250
}
4351

@@ -118,14 +126,8 @@ func (el *EventLoop) processChangesAndStatusUpdates(ctx context.Context, changes
118126
el.logger.Info("Processing a change",
119127
"host", c.Host.Value)
120128

121-
// TO-DO: This code is temporary. We will remove it once we have a component that processes changes.
122-
fmt.Printf("%+v\n", c)
123-
124129
if c.Op == state.Upsert {
125130
cfg, warnings := el.generator.GenerateForHost(c.Host)
126-
// TO-DO: for now, we only print the generated config, without writing it on the file system
127-
// and reloading NGINX.
128-
fmt.Println(string(cfg))
129131

130132
for obj, objWarnings := range warnings {
131133
for _, w := range objWarnings {
@@ -137,6 +139,28 @@ func (el *EventLoop) processChangesAndStatusUpdates(ctx context.Context, changes
137139
"warning", w)
138140
}
139141
}
142+
143+
el.logger.Info("Writing configuration",
144+
"host", c.Host.Value)
145+
146+
err := el.nginxFileMgr.WriteServerConfig(c.Host.Value, cfg)
147+
if err != nil {
148+
el.logger.Error(err, "Failed to write configuration",
149+
"host", c.Host.Value)
150+
}
151+
} else {
152+
err := el.nginxFileMgr.DeleteServerConfig(c.Host.Value)
153+
if err != nil {
154+
el.logger.Error(err, "Failed to delete configuration",
155+
"host", c.Host.Value)
156+
}
157+
}
158+
}
159+
160+
if len(changes) > 0 {
161+
err := el.nginxRuntimeMgr.Reload(ctx)
162+
if err != nil {
163+
el.logger.Error(err, "Failed to reload NGINX")
140164
}
141165
}
142166

‎internal/events/loop_test.go

Lines changed: 82 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@ import (
44
"context"
55

66
"github.com/nginxinc/nginx-gateway-kubernetes/internal/events"
7+
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/config"
78
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/config/configfakes"
9+
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/file/filefakes"
10+
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/runtime/runtimefakes"
811
"github.com/nginxinc/nginx-gateway-kubernetes/internal/state"
912
"github.com/nginxinc/nginx-gateway-kubernetes/internal/state/statefakes"
1013
"github.com/nginxinc/nginx-gateway-kubernetes/internal/status/statusfakes"
@@ -37,6 +40,8 @@ var _ = Describe("EventLoop", func() {
3740
var fakeUpdater *statusfakes.FakeUpdater
3841
var fakeServiceStore *statefakes.FakeServiceStore
3942
var fakeGenerator *configfakes.FakeGenerator
43+
var fakeNginxFimeMgr *filefakes.FakeManager
44+
var fakeNginxRuntimeMgr *runtimefakes.FakeManager
4045
var cancel context.CancelFunc
4146
var eventCh chan interface{}
4247
var errorCh chan error
@@ -47,7 +52,9 @@ var _ = Describe("EventLoop", func() {
4752
fakeUpdater = &statusfakes.FakeUpdater{}
4853
fakeServiceStore = &statefakes.FakeServiceStore{}
4954
fakeGenerator = &configfakes.FakeGenerator{}
50-
ctrl = events.NewEventLoop(fakeConf, fakeServiceStore, fakeGenerator, eventCh, fakeUpdater, zap.New())
55+
fakeNginxFimeMgr = &filefakes.FakeManager{}
56+
fakeNginxRuntimeMgr = &runtimefakes.FakeManager{}
57+
ctrl = events.NewEventLoop(fakeConf, fakeServiceStore, fakeGenerator, eventCh, fakeUpdater, zap.New(), fakeNginxFimeMgr, fakeNginxRuntimeMgr)
5158

5259
var ctx context.Context
5360

@@ -71,8 +78,10 @@ var _ = Describe("EventLoop", func() {
7178
It("should process upsert event", func() {
7279
fakeChanges := []state.Change{
7380
{
74-
Op: state.Upsert,
75-
Host: state.Host{},
81+
Op: state.Upsert,
82+
Host: state.Host{
83+
Value: "example.com",
84+
},
7685
},
7786
}
7887
fakeStatusUpdates := []state.StatusUpdate{
@@ -83,6 +92,9 @@ var _ = Describe("EventLoop", func() {
8392
}
8493
fakeConf.UpsertHTTPRouteReturns(fakeChanges, fakeStatusUpdates)
8594

95+
fakeCfg := []byte("fake")
96+
fakeGenerator.GenerateForHostReturns(fakeCfg, config.Warnings{})
97+
8698
hr := &v1alpha2.HTTPRoute{}
8799

88100
eventCh <- &events.UpsertEvent{
@@ -102,13 +114,22 @@ var _ = Describe("EventLoop", func() {
102114

103115
Eventually(fakeGenerator.GenerateForHostCallCount).Should(Equal(1))
104116
Expect(fakeGenerator.GenerateForHostArgsForCall(0)).Should(Equal(fakeChanges[0].Host))
117+
118+
Eventually(fakeNginxFimeMgr.WriteServerConfigCallCount).Should(Equal(1))
119+
host, cfg := fakeNginxFimeMgr.WriteServerConfigArgsForCall(0)
120+
Expect(host).Should(Equal("example.com"))
121+
Expect(cfg).Should(Equal(fakeCfg))
122+
123+
Eventually(fakeNginxRuntimeMgr.ReloadCallCount).Should(Equal(1))
105124
})
106125

107126
It("should process delete event", func() {
108127
fakeChanges := []state.Change{
109128
{
110-
Op: state.Delete,
111-
Host: state.Host{},
129+
Op: state.Delete,
130+
Host: state.Host{
131+
Value: "example.com",
132+
},
112133
},
113134
}
114135
fakeStatusUpdates := []state.StatusUpdate{
@@ -137,9 +158,10 @@ var _ = Describe("EventLoop", func() {
137158
return updates
138159
}).Should(Equal(fakeStatusUpdates))
139160

140-
// TO-DO:
141-
// once we have a component that processes host deletion, ensure that
142-
// its corresponding method is eventually called
161+
Eventually(fakeNginxFimeMgr.DeleteServerConfigCallCount).Should(Equal(1))
162+
Expect(fakeNginxFimeMgr.DeleteServerConfigArgsForCall(0)).Should(Equal("example.com"))
163+
164+
Eventually(fakeNginxRuntimeMgr.ReloadCallCount).Should(Equal(1))
143165
})
144166
})
145167

@@ -180,6 +202,58 @@ var _ = Describe("EventLoop", func() {
180202
})
181203
})
182204

205+
Describe("Processing events common cases", func() {
206+
AfterEach(func() {
207+
cancel()
208+
209+
var err error
210+
Eventually(errorCh).Should(Receive(&err))
211+
Expect(err).To(BeNil())
212+
})
213+
214+
It("should reload once in case of multiple changes", func() {
215+
fakeChanges := []state.Change{
216+
{
217+
Op: state.Delete,
218+
Host: state.Host{
219+
Value: "one.example.com",
220+
},
221+
},
222+
{
223+
Op: state.Upsert,
224+
Host: state.Host{
225+
Value: "two.example.com",
226+
},
227+
},
228+
}
229+
fakeConf.DeleteHTTPRouteReturns(fakeChanges, nil)
230+
231+
fakeCfg := []byte("fake")
232+
fakeGenerator.GenerateForHostReturns(fakeCfg, config.Warnings{})
233+
234+
nsname := types.NamespacedName{Namespace: "test", Name: "route"}
235+
236+
// the exact event doesn't matter. what matters is the generated changes return by DeleteHTTPRouteReturns
237+
eventCh <- &events.DeleteEvent{
238+
NamespacedName: nsname,
239+
Type: &v1alpha2.HTTPRoute{},
240+
}
241+
242+
Eventually(fakeConf.DeleteHTTPRouteCallCount).Should(Equal(1))
243+
Expect(fakeConf.DeleteHTTPRouteArgsForCall(0)).Should(Equal(nsname))
244+
245+
Eventually(fakeNginxFimeMgr.WriteServerConfigCallCount).Should(Equal(1))
246+
host, cfg := fakeNginxFimeMgr.WriteServerConfigArgsForCall(0)
247+
Expect(host).Should(Equal("two.example.com"))
248+
Expect(cfg).Should(Equal(fakeCfg))
249+
250+
Eventually(fakeNginxFimeMgr.DeleteServerConfigCallCount).Should(Equal(1))
251+
Expect(fakeNginxFimeMgr.DeleteServerConfigArgsForCall(0)).Should(Equal("one.example.com"))
252+
253+
Eventually(fakeNginxRuntimeMgr.ReloadCallCount).Should(Equal(1))
254+
})
255+
})
256+
183257
Describe("Edge cases", func() {
184258
AfterEach(func() {
185259
cancel()

‎internal/manager/manager.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212
hr "github.com/nginxinc/nginx-gateway-kubernetes/internal/implementations/httproute"
1313
svc "github.com/nginxinc/nginx-gateway-kubernetes/internal/implementations/service"
1414
ngxcfg "github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/config"
15+
"github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/file"
16+
ngxruntime "github.com/nginxinc/nginx-gateway-kubernetes/internal/nginx/runtime"
1517
"github.com/nginxinc/nginx-gateway-kubernetes/internal/state"
1618
"github.com/nginxinc/nginx-gateway-kubernetes/internal/status"
1719
nginxgwv1alpha1 "github.com/nginxinc/nginx-gateway-kubernetes/pkg/apis/gateway/v1alpha1"
@@ -78,7 +80,9 @@ func Start(cfg config.Config) error {
7880
serviceStore := state.NewServiceStore()
7981
reporter := status.NewUpdater(mgr.GetClient(), cfg.Logger)
8082
configGenerator := ngxcfg.NewGeneratorImpl(serviceStore)
81-
eventLoop := events.NewEventLoop(conf, serviceStore, configGenerator, eventCh, reporter, cfg.Logger)
83+
nginxFileMgr := file.NewManagerImpl()
84+
nginxRuntimeMgr := ngxruntime.NewManagerImpl()
85+
eventLoop := events.NewEventLoop(conf, serviceStore, configGenerator, eventCh, reporter, cfg.Logger, nginxFileMgr, nginxRuntimeMgr)
8286

8387
err = mgr.Add(eventLoop)
8488
if err != nil {

0 commit comments

Comments
 (0)