diff --git a/Makefile b/Makefile index 14759c41..a26ffe6c 100644 --- a/Makefile +++ b/Makefile @@ -237,17 +237,18 @@ generate-licenses: generate-dependency-notices ## Generates license/notice files # # We ignore the standard library (go list std) as a workaround for https://github.com/google/go-licenses/issues/244. # The awk script converts the output of `go list std` (line separated modules) to the input that `--ignore` expects +GO_LICENSES_STD_IGNORE := $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }'),internal/syscall/windows,internal/syscall/windows/registry,internal/syscall/windows/sysdll .PHONY: generate-dependency-notices generate-dependency-notices: go-licenses ifeq ($(detected_OS),windows) - $$env:GOOS="windows"; $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }') > NOTICE.windows - $$env:GOOS="darwin"; $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }') > NOTICE.darwin - $$env:GOOS="linux"; $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }') > NOTICE.linux + $$env:GOOS="windows"; $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(GO_LICENSES_STD_IGNORE) > NOTICE.windows + $$env:GOOS="darwin"; $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(GO_LICENSES_STD_IGNORE) > NOTICE.darwin + $$env:GOOS="linux"; $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(GO_LICENSES_STD_IGNORE) > NOTICE.linux $(CLEAR_GOARGS) $(GO_BIN) run scripts/notice.go else - GOOS="windows" $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }') > NOTICE.windows - GOOS="darwin" $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }') > NOTICE.darwin - GOOS="linux" $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(shell go list std | awk 'NR > 1 { printf(",") } { printf("%s",$$0) } END { print "" }') > NOTICE.linux + GOOS="windows" $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(GO_LICENSES_STD_IGNORE) > NOTICE.windows + GOOS="darwin" $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(GO_LICENSES_STD_IGNORE) > NOTICE.darwin + GOOS="linux" $(GO_LICENSES) report ./cmd/dcp --template NOTICE.tmpl --ignore github.com/microsoft/dcp --ignore $(GO_LICENSES_STD_IGNORE) > NOTICE.linux $(CLEAR_GOARGS) $(GO_BIN) run scripts/notice.go endif diff --git a/NOTICE b/NOTICE index 53d72bb2..ed7dba0b 100644 --- a/NOTICE +++ b/NOTICE @@ -1047,6 +1047,35 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ---------------------------------------------------------- +github.com/dustin/go-humanize v1.0.1 - MIT +https://github.com/dustin/go-humanize/blob/v1.0.1/LICENSE + +Copyright (c) 2005-2008 Dustin Sallings + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +---------------------------------------------------------- + +---------------------------------------------------------- + github.com/ebitengine/purego v0.10.0 - Apache-2.0 https://github.com/ebitengine/purego/blob/v0.10.0/LICENSE @@ -5081,6 +5110,42 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------- +github.com/golang-migrate/migrate/v4 v4.19.1 - MIT +https://github.com/golang-migrate/migrate/blob/v4.19.1/LICENSE + +The MIT License (MIT) + +Original Work +Copyright (c) 2016 Matthias Kadenbach +https://github.com/mattes/migrate + +Modified Work +Copyright (c) 2018 Dale Hui +https://github.com/golang-migrate/migrate + + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------- + +---------------------------------------------------------- + github.com/golang/protobuf/proto v1.5.4 - BSD-3-Clause https://github.com/golang/protobuf/blob/v1.5.4/LICENSE @@ -6564,6 +6629,23 @@ https://github.com/kylelemons/godebug/blob/v1.1.0/LICENSE ---------------------------------------------------------- +github.com/mattn/go-isatty v0.0.20 - MIT +https://github.com/mattn/go-isatty/blob/v0.0.20/LICENSE + +Copyright (c) Yasuhiro MATSUMOTO + +MIT License (Expat) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------- + +---------------------------------------------------------- + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd - Apache-2.0 https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE @@ -7021,6 +7103,35 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------- +github.com/ncruces/go-strftime v1.0.0 - MIT +https://github.com/ncruces/go-strftime/blob/v1.0.0/LICENSE + +MIT License + +Copyright (c) 2022 Nuno Cruces + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------- + +---------------------------------------------------------- + github.com/pmezard/go-difflib/difflib v1.0.1-0.20181226105442-5d4384ee4fb2 - BSD-3-Clause https://github.com/pmezard/go-difflib/blob/5d4384ee4fb2/LICENSE @@ -7927,6 +8038,41 @@ https://github.com/prometheus/procfs/blob/v0.20.1/LICENSE ---------------------------------------------------------- +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec - BSD-3-Clause +https://github.com/remyoudompheng/bigfft/blob/24d4a6f8daec/LICENSE + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------- + +---------------------------------------------------------- + github.com/shirou/gopsutil/v4 v4.26.4 - BSD-3-Clause https://github.com/shirou/gopsutil/blob/v4.26.4/LICENSE @@ -17105,6 +17251,145 @@ https://github.com/kubernetes/utils/blob/28399d86e0b5/third_party/forked/golang/ ---------------------------------------------------------- +modernc.org/libc v1.72.3 - BSD-3-Clause +https://gitlab.com/cznic/libc/-/blob/v1.72.3/LICENSE + +Copyright (c) 2017 The Libc Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------- + +---------------------------------------------------------- + +modernc.org/mathutil v1.7.1 - BSD-3-Clause +https://gitlab.com/cznic/mathutil/-/blob/v1.7.1/LICENSE + +Copyright (c) 2014 The mathutil Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------- + +---------------------------------------------------------- + +modernc.org/memory v1.11.0 - BSD-3-Clause +https://gitlab.com/cznic/memory/-/blob/v1.11.0/LICENSE + +Copyright (c) 2017 The Memory Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the names of the authors nor the names of the +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------- + +---------------------------------------------------------- + +modernc.org/sqlite v1.50.1 - BSD-3-Clause +https://gitlab.com/cznic/sqlite/-/blob/v1.50.1/LICENSE + +Copyright (c) 2017 The Sqlite Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------- + +---------------------------------------------------------- + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 - Apache-2.0 https://github.com/kubernetes-sigs/apiserver-network-proxy/blob/konnectivity-client/v0.34.0/konnectivity-client/LICENSE diff --git a/README.md b/README.md index 70f5fcc9..ee4802db 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ DCP has knowledge of a number of environment variables that can change its behav | `DCP_EXTENSIONS_PATH` | Points to directory that contains DCP extensions. By default extensions are placed in the `ext` sub-directory of the directory where DCP main executable is located. | | `DEBUG_SESSION_PORT`, `DEBUG_SESSION_TOKEN`, and `DEBUG_SESSION_SERVER_CERTIFICATE` | These are variables that configure the endpoint for running Executables via a developer IDE/under debugger. For more information see [IDE execution specification](https://github.com/dotnet/aspire/blob/main/docs/specs/IDE-execution.md). | | `DCP_SESSION_FOLDER` | This variable is used for isolating multiple DCP instances running concurrently on the same machine. If set (to a valid filesystem folder), DCP process(es) will create files related to their execution in this folder: the access configuration file (kubeconfig), captured Executable/Container logs, etc. | +| `DCP_STATE_STORE_PATH` | Overrides the path to the local SQLite state store used for DCP coordination metadata such as process records and resource leases. Resource leases are owned by the leasing DCP process identity and stale leases are cleaned up on startup. If unset, DCP uses the default per-user state store under a standard or elevated subdirectory of the DCP user directory. | | `DCP_LOG_SOCKET` | If set to a Unix domain socket, DCP will write its execution logs to that socket instead of writing them to standard error stream (`stderr`). This allows programs that launch DCP to capture its output even if DCP is running in `--detach` mode.
The `--detach` mode causes DCP to fork itself and break the parent-child relationship (and lifetime dependency) from the process that launched it, but the side effect of doing so is that the parent process loses ability to monitor DCP standard output and standard error streamd. | | `DCP_LOG_SESSION_ID` | If set, DCP will prepend this value to all diagnostics log names. If unset, a session ID will be calculated. The value is propagated to all child DCP processes. | | `DCP_DIAGNOSTICS_LOG_LEVEL` | If set, enabled DCP diagnostic logging.
Can be set to `error`, `info`, or `debug`; for troubleshooting `debug` is recommended, although it results in the most verbose output. | diff --git a/api/v1/common_types.go b/api/v1/common_types.go index 48e2315e..7e909036 100644 --- a/api/v1/common_types.go +++ b/api/v1/common_types.go @@ -6,7 +6,10 @@ package v1 import ( + "encoding/gob" "fmt" + "io" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -25,6 +28,30 @@ type EnvVar struct { // CONSIDER allowing expansion of existing variable references e.g. using ${VAR_NAME} syntax and $$ to escape the $ sign } +// To get consistent output from gob encoders, we need to introduce types in +// a deterministic order as the encoder generates (and globally caches) an incrementing ID +// for each type it encounters. This is a bit of a hack, but it works. +// Any types being encoded in lifecycle GetLifecycleKey methods need to be registered here. +func initializeLifecycleHashEncoder() { + initEncoder := gob.NewEncoder(io.Discard) + + _ = initEncoder.Encode(ContainerLabel{}) + _ = initEncoder.Encode(ContainerBuildSecret{}) + _ = initEncoder.Encode(VolumeMount{}) + _ = initEncoder.Encode(ContainerPort{}) + _ = initEncoder.Encode(EnvVar{}) + _ = initEncoder.Encode(CreateFileSystem{}) + _ = initEncoder.Encode(ContainerPemCertificates{}) + _ = initEncoder.Encode(ImageLayer{}) + + _ = initEncoder.Encode(time.Time{}) + _ = initEncoder.Encode(ExecutablePemCertificates{}) +} + +func init() { + initializeLifecycleHashEncoder() +} + const LogSubresourceName = "log" // MaxAnnotationsTotalSize is the maximum total size of all annotations in bytes. diff --git a/api/v1/container_network_types.go b/api/v1/container_network_types.go index de82445b..9b5e3ccd 100644 --- a/api/v1/container_network_types.go +++ b/api/v1/container_network_types.go @@ -7,7 +7,10 @@ package v1 import ( "context" + "fmt" + "strings" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/pkg/commonapi" apiserver_resource "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource" apiserver_resourcerest "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource/resourcerest" @@ -107,6 +110,10 @@ func (cn *ContainerNetwork) GetGroupVersionResource() schema.GroupVersionResourc } } +func (cn *ContainerNetwork) GetLeaseKey() string { + return fmt.Sprintf("%s/%s", cn.GetGroupVersionResource().Resource, strings.TrimSpace(cn.Spec.NetworkName)) +} + func (cn *ContainerNetwork) GetObjectMeta() *metav1.ObjectMeta { return &cn.ObjectMeta } @@ -214,3 +221,4 @@ var _ apiserver_resource.StatusSubResource = (*ContainerNetworkStatus)(nil) var _ apiserver_resourcerest.ShortNamesProvider = (*ContainerNetwork)(nil) var _ apiserver_resourcestrategy.Validater = (*ContainerNetwork)(nil) var _ apiserver_resourcestrategy.ValidateUpdater = (*ContainerNetwork)(nil) +var _ statestore.LeasableResource = (*ContainerNetwork)(nil) diff --git a/api/v1/container_types.go b/api/v1/container_types.go index d085ec59..fbb196b8 100644 --- a/api/v1/container_types.go +++ b/api/v1/container_types.go @@ -12,7 +12,6 @@ import ( "errors" "fmt" "hash/fnv" - "io" "io/fs" "os" "path" @@ -34,7 +33,9 @@ import ( apiserver_resourcerest "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource/resourcerest" apiserver_resourcestrategy "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource/resourcestrategy" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/pkg/commonapi" + "github.com/microsoft/dcp/pkg/osutil" "github.com/microsoft/dcp/pkg/pointers" ) @@ -652,6 +653,15 @@ type ContainerSpec struct { // Should this container be created and persisted between DCP runs? Persistent bool `json:"persistent,omitempty"` + // Optional parent process PID used to scope persistent Container cleanup to a process lifecycle. + // When set, MonitorTimestamp must also be set and Persistent must be true. + // +optional + MonitorPID *int64 `json:"monitorPid,omitempty"` + + // Optional parent process identity timestamp used with MonitorPID to guard against PID reuse. + // +optional + MonitorTimestamp metav1.MicroTime `json:"monitorTimestamp,omitempty"` + // Additional arguments to pass to the container run command // +listType=atomic RunArgs []string `json:"runArgs,omitempty"` @@ -762,6 +772,14 @@ func (cs *ContainerSpec) Equal(other *ContainerSpec) bool { return false } + if !pointers.EqualValue(cs.MonitorPID, other.MonitorPID) { + return false + } + + if !osutil.MicroEqual(cs.MonitorTimestamp, other.MonitorTimestamp) { + return false + } + if !slices.Equal(cs.RunArgs, other.RunArgs) { return false } @@ -797,23 +815,6 @@ func (cs *ContainerSpec) Equal(other *ContainerSpec) bool { return true } -// To get consistent output from gob encoders, we need to introduce types in -// a deterministic order as the encoder generates (and globally caches) an incrementing ID -// for each type it encounters. This is a bit of a hack, but it works. -// Any types being encoded in GetLifecycleKey need to be registered here. -func initializeHashEncoder() { - initEncoder := gob.NewEncoder(io.Discard) - - _ = initEncoder.Encode(ContainerLabel{}) - _ = initEncoder.Encode(ContainerBuildSecret{}) - _ = initEncoder.Encode(VolumeMount{}) - _ = initEncoder.Encode(ContainerPort{}) - _ = initEncoder.Encode(EnvVar{}) - _ = initEncoder.Encode(CreateFileSystem{}) - _ = initEncoder.Encode(ContainerPemCertificates{}) - _ = initEncoder.Encode(ImageLayer{}) -} - func (cs *ContainerSpec) GetLifecycleKey() (string, bool, error) { if cs.LifecycleKey != "" { return cs.LifecycleKey, false, nil @@ -971,6 +972,11 @@ func (cs *ContainerSpec) GetLifecycleKey() (string, bool, error) { } } + if cs.MonitorPID != nil { + hashErr = errors.Join(hashErr, encoder.Encode(*cs.MonitorPID)) + hashErr = errors.Join(hashErr, encoder.Encode(cs.MonitorTimestamp.Time)) + } + if len(cs.CreateFiles) > 0 { // Add the create files to the hash sortedCreateFiles := slices.Clone(cs.CreateFiles) @@ -1175,6 +1181,10 @@ func (c *Container) GetGroupVersionResource() schema.GroupVersionResource { } } +func (c *Container) GetLeaseKey() string { + return fmt.Sprintf("%s/%s", c.GetGroupVersionResource().Resource, strings.TrimSpace(c.Spec.ContainerName)) +} + func (c *Container) GetObjectMeta() *metav1.ObjectMeta { return &c.ObjectMeta } @@ -1276,6 +1286,23 @@ func (c *Container) Validate(ctx context.Context) field.ErrorList { errorList = append(errorList, field.Required(field.NewPath("spec", "containerName"), "containerName must be set to a value when persistent is true")) } + monitorTimestampSet := !c.Spec.MonitorTimestamp.IsZero() + if c.Spec.MonitorPID != nil && *c.Spec.MonitorPID <= 0 { + errorList = append(errorList, field.Invalid(field.NewPath("spec", "monitorPid"), *c.Spec.MonitorPID, "monitorPid must be positive")) + } + if !c.Spec.Persistent && c.Spec.MonitorPID != nil { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "monitorPid"), "monitorPid can only be set when persistent is true")) + } + if !c.Spec.Persistent && monitorTimestampSet { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "monitorTimestamp"), "monitorTimestamp can only be set when persistent is true")) + } + if c.Spec.MonitorPID != nil && !monitorTimestampSet { + errorList = append(errorList, field.Required(field.NewPath("spec", "monitorTimestamp"), "monitorTimestamp must be set when monitorPid is set")) + } + if c.Spec.MonitorPID == nil && monitorTimestampSet { + errorList = append(errorList, field.Required(field.NewPath("spec", "monitorPid"), "monitorPid must be set when monitorTimestamp is set")) + } + healthProbesPath := field.NewPath("spec", "healthProbes") for i, probe := range c.Spec.HealthProbes { errorList = append(errorList, probe.Validate(healthProbesPath.Index(i))...) @@ -1365,6 +1392,14 @@ func (c *Container) ValidateUpdate(ctx context.Context, obj runtime.Object) fiel errorList = append(errorList, field.Forbidden(field.NewPath("spec", "persistent"), "persistent cannot be changed")) } + if !pointers.EqualValue(oldContainer.Spec.MonitorPID, c.Spec.MonitorPID) { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "monitorPid"), "monitorPid cannot be changed")) + } + + if !osutil.MicroEqual(oldContainer.Spec.MonitorTimestamp, c.Spec.MonitorTimestamp) { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "monitorTimestamp"), "monitorTimestamp cannot be changed")) + } + // Forbid changing labels after the resource is created if !slices.Equal(oldContainer.Spec.Labels, c.Spec.Labels) { errorList = append(errorList, field.Forbidden(field.NewPath("spec", "labels"), "labels cannot be changed")) @@ -1536,7 +1571,6 @@ func (clr *ContainerLogResource) GetStorageProvider( func init() { SchemeBuilder.Register(&Container{}, &ContainerList{}) - initializeHashEncoder() } // Ensure types support interfaces expected by our API server @@ -1550,3 +1584,4 @@ var _ apiserver_resourcestrategy.Validater = (*Container)(nil) var _ apiserver_resourcestrategy.ValidateUpdater = (*Container)(nil) var _ apiserver_resource.ObjectWithGenericSubResource = (*Container)(nil) var _ apiserver_resource.GenericSubResource = (*ContainerLogResource)(nil) +var _ statestore.LeasableResource = (*Container)(nil) diff --git a/api/v1/container_types_test.go b/api/v1/container_types_test.go index 972e64dd..28db68bf 100644 --- a/api/v1/container_types_test.go +++ b/api/v1/container_types_test.go @@ -7,12 +7,145 @@ package v1 import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" ) +func TestContainerGetLeaseKey(t *testing.T) { + t.Parallel() + + container := &Container{} + container.Spec.ContainerName = " api " + + require.Equal(t, "containers/api", container.GetLeaseKey()) +} + +func TestContainerNetworkGetLeaseKey(t *testing.T) { + t.Parallel() + + network := &ContainerNetwork{} + network.Spec.NetworkName = " app-network " + + require.Equal(t, "containernetworks/app-network", network.GetLeaseKey()) +} + +func TestContainerSpecGetLifecycleKeyIncludesMonitorFields(t *testing.T) { + t.Parallel() + + monitorPID := int64(12345) + monitorTimestamp := metav1.NewMicroTime(time.Now().UTC()) + spec := &ContainerSpec{ + Image: "api:dev", + ContainerName: "api", + Persistent: true, + } + + key, _, keyErr := spec.GetLifecycleKey() + require.NoError(t, keyErr) + + spec.MonitorPID = &monitorPID + spec.MonitorTimestamp = monitorTimestamp + keyWithMonitor, _, monitorKeyErr := spec.GetLifecycleKey() + require.NoError(t, monitorKeyErr) + require.NotEqual(t, key, keyWithMonitor) + + differentMonitorPID := monitorPID + 1 + spec.MonitorPID = &differentMonitorPID + keyWithDifferentMonitorPID, _, differentMonitorPIDErr := spec.GetLifecycleKey() + require.NoError(t, differentMonitorPIDErr) + require.NotEqual(t, keyWithMonitor, keyWithDifferentMonitorPID) + + spec.MonitorPID = &monitorPID + spec.MonitorTimestamp = metav1.NewMicroTime(monitorTimestamp.Time.Add(time.Second)) + keyWithDifferentMonitorTimestamp, _, differentMonitorTimestampErr := spec.GetLifecycleKey() + require.NoError(t, differentMonitorTimestampErr) + require.NotEqual(t, keyWithMonitor, keyWithDifferentMonitorTimestamp) +} + +func TestContainerValidateMonitorFields(t *testing.T) { + t.Parallel() + + monitorPID := int64(12345) + negativeMonitorPID := int64(-1) + monitorTimestamp := metav1.NewMicroTime(time.Now().UTC()) + + testCases := []struct { + name string + spec ContainerSpec + expectError bool + }{ + { + name: "valid persistent monitor", + spec: ContainerSpec{ + Image: "api:dev", + ContainerName: "api", + Persistent: true, + MonitorPID: &monitorPID, + MonitorTimestamp: monitorTimestamp, + }, + }, + { + name: "monitor requires persistent", + spec: ContainerSpec{ + Image: "api:dev", + MonitorPID: &monitorPID, + MonitorTimestamp: monitorTimestamp, + }, + expectError: true, + }, + { + name: "monitor pid requires timestamp", + spec: ContainerSpec{ + Image: "api:dev", + ContainerName: "api", + Persistent: true, + MonitorPID: &monitorPID, + }, + expectError: true, + }, + { + name: "monitor timestamp requires pid", + spec: ContainerSpec{ + Image: "api:dev", + ContainerName: "api", + Persistent: true, + MonitorTimestamp: monitorTimestamp, + }, + expectError: true, + }, + { + name: "monitor pid must be positive", + spec: ContainerSpec{ + Image: "api:dev", + ContainerName: "api", + Persistent: true, + MonitorPID: &negativeMonitorPID, + MonitorTimestamp: monitorTimestamp, + }, + expectError: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + container := &Container{Spec: testCase.spec} + errors := container.Validate(nil) + + if testCase.expectError { + require.NotEmpty(t, errors) + } else { + require.Empty(t, errors) + } + }) + } +} + func TestImageLayerValidate(t *testing.T) { t.Parallel() diff --git a/api/v1/executable_types.go b/api/v1/executable_types.go index 765935d8..6924e50c 100644 --- a/api/v1/executable_types.go +++ b/api/v1/executable_types.go @@ -7,10 +7,16 @@ package v1 import ( "context" + "encoding/gob" + "errors" "fmt" + "hash/fnv" stdmaps "maps" + "os" stdslices "slices" + "strings" + "github.com/joho/godotenv" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -24,7 +30,11 @@ import ( apiserver_resourcerest "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource/resourcerest" apiserver_resourcestrategy "github.com/tilt-dev/tilt-apiserver/pkg/server/builder/resource/resourcestrategy" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/pkg/commonapi" + usvc_maps "github.com/microsoft/dcp/pkg/maps" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/pointers" "github.com/microsoft/dcp/pkg/slices" ) @@ -250,10 +260,30 @@ type ExecutableSpec struct { // Controls behavior of environment variables inherited from the controller process. AmbientEnvironment AmbientEnvironment `json:"ambientEnvironment,omitempty"` + // Should the controller attempt to start the Executable? + // +kubebuilder:default:=true + Start *bool `json:"start,omitempty"` + // Should the controller attempt to stop the Executable // +kubebuilder:default:=false Stop bool `json:"stop,omitempty"` + // Should this Executable be created and persisted between DCP runs? + Persistent bool `json:"persistent,omitempty"` + + // Optional key used to identify if an existing persistent Executable process should be reused. + // If not set, the controller will calculate a key based on a hash of specific fields in the ExecutableSpec. + LifecycleKey string `json:"lifecycleKey,omitempty"` + + // Optional parent process PID used to scope persistent Executable cleanup to a process lifecycle. + // When set, MonitorTimestamp must also be set and Persistent must be true. + // +optional + MonitorPID *int64 `json:"monitorPid,omitempty"` + + // Optional parent process identity timestamp used with MonitorPID to guard against PID reuse. + // +optional + MonitorTimestamp metav1.MicroTime `json:"monitorTimestamp,omitempty"` + // Health probe configuration for the Executable // +listType=atomic HealthProbes []HealthProbe `json:"healthProbes,omitempty"` @@ -296,10 +326,30 @@ func (es ExecutableSpec) Equal(other ExecutableSpec) bool { return false } + if pointers.GetValueOrDefault(es.Start, true) != pointers.GetValueOrDefault(other.Start, true) { + return false + } + if es.Stop != other.Stop { return false } + if es.Persistent != other.Persistent { + return false + } + + if es.LifecycleKey != other.LifecycleKey { + return false + } + + if !pointers.EqualValue(es.MonitorPID, other.MonitorPID) { + return false + } + + if !osutil.MicroEqual(es.MonitorTimestamp, other.MonitorTimestamp) { + return false + } + if len(es.HealthProbes) != len(other.HealthProbes) { return false } @@ -317,6 +367,168 @@ func (es ExecutableSpec) Equal(other ExecutableSpec) bool { return true } +func (es *ExecutableSpec) GetLifecycleKey() (string, bool, error) { + if es.LifecycleKey != "" { + return es.LifecycleKey, false, nil + } + + fnvHash := fnv.New128() + encoder := gob.NewEncoder(fnvHash) + + var hashErr error + hashErr = errors.Join(hashErr, encoder.Encode(es.ExecutablePath)) + hashErr = errors.Join(hashErr, encoder.Encode(es.WorkingDirectory)) + hashErr = errors.Join(hashErr, encoder.Encode(string(es.ExecutionType))) + hashErr = errors.Join(hashErr, encoder.Encode(string(es.AmbientEnvironment.Behavior))) + hashErr = errors.Join(hashErr, encoder.Encode(es.Args)) + if es.MonitorPID != nil { + hashErr = errors.Join(hashErr, encoder.Encode(*es.MonitorPID)) + hashErr = errors.Join(hashErr, encoder.Encode(es.MonitorTimestamp.Time)) + } + + if len(es.Env) > 0 { + sortedEnv := stdslices.Clone(es.Env) + stdslices.SortFunc(sortedEnv, func(e1, e2 EnvVar) int { + return strings.Compare(e1.Name, e2.Name) + }) + + for i := range sortedEnv { + hashErr = errors.Join(hashErr, encoder.Encode(sortedEnv[i])) + } + } + + if len(es.EnvFiles) > 0 { + sortedEnvFiles := stdslices.Clone(es.EnvFiles) + stdslices.Sort(sortedEnvFiles) + + for i := range sortedEnvFiles { + envFileContents, envFileReadErr := os.ReadFile(sortedEnvFiles[i]) + if envFileReadErr != nil { + hashErr = errors.Join(hashErr, envFileReadErr) + } else { + hashErr = errors.Join(hashErr, encoder.Encode(envFileContents)) + } + } + } + + if es.PemCertificates != nil { + sortedPemCertificates := stdslices.Clone(es.PemCertificates.Certificates) + stdslices.SortFunc(sortedPemCertificates, func(c1, c2 PemCertificate) int { + return strings.Compare(c1.Thumbprint, c2.Thumbprint) + }) + + for i := range sortedPemCertificates { + hashErr = errors.Join(hashErr, encoder.Encode(sortedPemCertificates[i])) + } + hashErr = errors.Join(hashErr, encoder.Encode(es.PemCertificates.ContinueOnError)) + } + + lifecycleKey := fmt.Sprintf("%x", fnvHash.Sum(nil)) + return lifecycleKey, true, hashErr +} + +func (e *Executable) GetLifecycleKey() (string, bool, error) { + lifecycleSpec, lifecycleSpecErr := e.EffectiveLifecycleSpec() + if lifecycleSpecErr != nil { + return "", false, lifecycleSpecErr + } + return lifecycleSpec.GetLifecycleKey() +} + +func (e *Executable) EffectiveLifecycleSpec() (ExecutableSpec, error) { + lifecycleSpec := *e.Spec.DeepCopy() + effectiveArgs, effectiveArgsErr := effectiveLifecycleArgs(e) + if effectiveArgsErr != nil { + return ExecutableSpec{}, effectiveArgsErr + } + explicitEffectiveEnv, explicitEffectiveEnvErr := explicitEffectiveLifecycleEnv(e) + if explicitEffectiveEnvErr != nil { + return ExecutableSpec{}, explicitEffectiveEnvErr + } + lifecycleSpec.Args = effectiveArgs + lifecycleSpec.Env = explicitEffectiveEnv + lifecycleSpec.EnvFiles = nil + return lifecycleSpec, nil +} + +func effectiveLifecycleArgs(e *Executable) ([]string, error) { + if len(e.Status.EffectiveArgs) == 0 { + if len(e.Spec.Args) > 0 { + return nil, fmt.Errorf("executable lifecycle key cannot be calculated before effective arguments are computed") + } + return nil, nil + } + return stdslices.Clone(e.Status.EffectiveArgs), nil +} + +func explicitEffectiveLifecycleEnv(e *Executable) ([]EnvVar, error) { + explicitNames, explicitNamesErr := explicitLifecycleEnvNames(e) + if explicitNamesErr != nil { + return nil, explicitNamesErr + } + if explicitNames.Len() == 0 { + return nil, nil + } + + effectiveEnvByName := lifecycleEnvMap() + for _, envVar := range e.Status.EffectiveEnv { + effectiveEnvByName.Set(envVar.Name, envVar.Value) + } + if effectiveEnvByName.Len() == 0 { + return nil, fmt.Errorf("executable lifecycle key cannot be calculated before effective environment is computed") + } + + explicitEffectiveEnv := make([]EnvVar, 0, explicitNames.Len()) + for nameKey, name := range explicitNames.Data() { + value, found := effectiveEnvByName.Get(nameKey) + if !found { + continue + } + explicitEffectiveEnv = append(explicitEffectiveEnv, EnvVar{Name: name, Value: value}) + } + stdslices.SortFunc(explicitEffectiveEnv, func(e1, e2 EnvVar) int { + return strings.Compare(lifecycleEnvKey(e1.Name), lifecycleEnvKey(e2.Name)) + }) + return explicitEffectiveEnv, nil +} + +func explicitLifecycleEnvNames(e *Executable) (usvc_maps.StringKeyMap[string], error) { + explicitNames := lifecycleEnvMap() + addExplicitName := func(name string) { + if name != "" { + explicitNames.Set(name, name) + } + } + + if len(e.Spec.EnvFiles) > 0 { + fileEnv, readErr := godotenv.Read(e.Spec.EnvFiles...) + if readErr != nil { + return explicitNames, fmt.Errorf("could not read executable environment files for lifecycle key: %w", readErr) + } + for name := range fileEnv { + addExplicitName(name) + } + } + for _, envVar := range e.Spec.Env { + addExplicitName(envVar.Name) + } + return explicitNames, nil +} + +func lifecycleEnvMap() usvc_maps.StringKeyMap[string] { + if osutil.IsWindows() { + return usvc_maps.NewStringKeyMap[string](usvc_maps.StringMapModeCaseInsensitive) + } + return usvc_maps.NewStringKeyMap[string](usvc_maps.StringMapModeCaseSensitive) +} + +func lifecycleEnvKey(name string) string { + if osutil.IsWindows() { + return strings.ToUpper(name) + } + return name +} + func (es ExecutableSpec) Validate(specPath *field.Path) field.ErrorList { errorList := field.ErrorList{} @@ -348,6 +560,34 @@ func (es ExecutableSpec) Validate(specPath *field.Path) field.ErrorList { errorList = append(errorList, field.Invalid(specPath.Child("ambientEnvironment", "behavior"), es.AmbientEnvironment.Behavior, "Ambient environment behavior must be either Inherit or DoNotInherit.")) } + effectiveExecutionType := es.ExecutionType + if effectiveExecutionType == "" { + effectiveExecutionType = ExecutionTypeProcess + } + if es.Persistent && effectiveExecutionType != ExecutionTypeProcess { + errorList = append(errorList, field.Invalid(specPath.Child("persistent"), es.Persistent, "Persistent Executables only support Process execution type.")) + } + if es.Persistent && len(es.FallbackExecutionTypes) > 0 { + errorList = append(errorList, field.Invalid(specPath.Child("fallbackExecutionTypes"), es.FallbackExecutionTypes, "Persistent Executables cannot use fallback execution types.")) + } + + monitorTimestampSet := !es.MonitorTimestamp.IsZero() + if es.MonitorPID != nil && *es.MonitorPID <= 0 { + errorList = append(errorList, field.Invalid(specPath.Child("monitorPid"), *es.MonitorPID, "monitorPid must be positive")) + } + if !es.Persistent && es.MonitorPID != nil { + errorList = append(errorList, field.Forbidden(specPath.Child("monitorPid"), "monitorPid can only be set when persistent is true")) + } + if !es.Persistent && monitorTimestampSet { + errorList = append(errorList, field.Forbidden(specPath.Child("monitorTimestamp"), "monitorTimestamp can only be set when persistent is true")) + } + if es.MonitorPID != nil && !monitorTimestampSet { + errorList = append(errorList, field.Required(specPath.Child("monitorTimestamp"), "monitorTimestamp must be set when monitorPid is set")) + } + if es.MonitorPID == nil && monitorTimestampSet { + errorList = append(errorList, field.Required(specPath.Child("monitorPid"), "monitorPid must be set when monitorTimestamp is set")) + } + healthProbesPath := specPath.Child("healthProbes") for i, probe := range es.HealthProbes { errorList = append(errorList, probe.Validate(healthProbesPath.Index(i))...) @@ -453,6 +693,10 @@ func (e *Executable) GetResourceId() string { return fmt.Sprintf("executable-%s", e.UID) } +func (e *Executable) GetLeaseKey() string { + return e.NamespacedName().String() +} + func (e *Executable) GetGroupVersionResource() schema.GroupVersionResource { return schema.GroupVersionResource{ Group: GroupVersion.Group, @@ -517,10 +761,26 @@ func (e *Executable) ValidateUpdate(ctx context.Context, obj runtime.Object) fie errorList := field.ErrorList{} oldExe := obj.(*Executable) + if (oldExe.Spec.Start == nil || *oldExe.Spec.Start) && (e.Spec.Start != nil && !*e.Spec.Start) { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "start"), "Cannot set start to false after Executable creation.")) + } + if oldExe.Spec.Stop && e.Spec.Stop != oldExe.Spec.Stop { errorList = append(errorList, field.Forbidden(field.NewPath("spec", "stop"), "Cannot unset stop property once it is set.")) } + if oldExe.Spec.Persistent != e.Spec.Persistent { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "persistent"), "persistent cannot be changed")) + } + + if !pointers.EqualValue(oldExe.Spec.MonitorPID, e.Spec.MonitorPID) { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "monitorPid"), "monitorPid cannot be changed")) + } + + if !osutil.MicroEqual(oldExe.Spec.MonitorTimestamp, e.Spec.MonitorTimestamp) { + errorList = append(errorList, field.Forbidden(field.NewPath("spec", "monitorTimestamp"), "monitorTimestamp cannot be changed")) + } + if oldExe.Spec.AmbientEnvironment.Behavior != e.Spec.AmbientEnvironment.Behavior { errorList = append(errorList, field.Forbidden(field.NewPath("spec", "ambientEnvironment", "behavior"), "Cannot change ambient environment behavior once it is set.")) } @@ -547,6 +807,10 @@ func (e *Executable) Done() bool { return !e.Status.FinishTimestamp.IsZero() } +func (e *Executable) ShouldStart() bool { + return e.Spec.Start == nil || *e.Spec.Start +} + func (*Executable) GenericSubResources() []apiserver_resource.GenericSubResource { return []apiserver_resource.GenericSubResource{ &ExecutableLogResource{}, @@ -630,3 +894,4 @@ var _ apiserver_resourcestrategy.ValidateUpdater = (*Executable)(nil) var _ apiserver_resource.ObjectWithGenericSubResource = (*Executable)(nil) var _ apiserver_resource.GenericSubResource = (*ExecutableLogResource)(nil) var _ StdIoStreamableResource = (*Executable)(nil) +var _ statestore.LeasableResource = (*Executable)(nil) diff --git a/api/v1/executable_types_test.go b/api/v1/executable_types_test.go new file mode 100644 index 00000000..a7a7e523 --- /dev/null +++ b/api/v1/executable_types_test.go @@ -0,0 +1,414 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package v1 + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestExecutableShouldStart(t *testing.T) { + t.Parallel() + + shouldStart := true + shouldNotStart := false + + testCases := []struct { + name string + start *bool + expected bool + }{ + { + name: "omitted", + start: nil, + expected: true, + }, + { + name: "explicit true", + start: &shouldStart, + expected: true, + }, + { + name: "explicit false", + start: &shouldNotStart, + expected: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + exe := Executable{ + Spec: ExecutableSpec{ + Start: testCase.start, + }, + } + + assert.Equal(t, testCase.expected, exe.ShouldStart()) + }) + } +} + +func TestExecutableGetLeaseKey(t *testing.T) { + t.Parallel() + + exe := &Executable{} + exe.Namespace = "default" + exe.Name = "api" + + require.Equal(t, "default/api", exe.GetLeaseKey()) +} + +func TestExecutableSpecGetLifecycleKeyPreservesStringBoundaries(t *testing.T) { + t.Parallel() + + spec := ExecutableSpec{ + ExecutablePath: "A", + WorkingDirectory: "BC", + Args: []string{"A", "BC"}, + } + + key, computed, keyErr := spec.GetLifecycleKey() + require.NoError(t, keyErr) + require.True(t, computed) + + specWithAmbiguousConcatenation := ExecutableSpec{ + ExecutablePath: "AB", + WorkingDirectory: "C", + Args: []string{"AB", "C"}, + } + + keyWithAmbiguousConcatenation, _, ambiguousConcatenationErr := specWithAmbiguousConcatenation.GetLifecycleKey() + require.NoError(t, ambiguousConcatenationErr) + require.NotEqual(t, key, keyWithAmbiguousConcatenation) +} + +func TestExecutableSpecGetLifecycleKeyIncludesMonitorFields(t *testing.T) { + t.Parallel() + + monitorPID := int64(12345) + monitorTimestamp := metav1.NewMicroTime(time.Now().UTC()) + spec := ExecutableSpec{ + ExecutablePath: "/path/to/app", + WorkingDirectory: "/path/to/workdir", + Persistent: true, + } + + key, _, keyErr := spec.GetLifecycleKey() + require.NoError(t, keyErr) + + spec.MonitorPID = &monitorPID + spec.MonitorTimestamp = monitorTimestamp + keyWithMonitor, _, monitorKeyErr := spec.GetLifecycleKey() + require.NoError(t, monitorKeyErr) + require.NotEqual(t, key, keyWithMonitor) + + differentMonitorPID := monitorPID + 1 + spec.MonitorPID = &differentMonitorPID + keyWithDifferentMonitorPID, _, differentMonitorPIDErr := spec.GetLifecycleKey() + require.NoError(t, differentMonitorPIDErr) + require.NotEqual(t, keyWithMonitor, keyWithDifferentMonitorPID) + + spec.MonitorPID = &monitorPID + spec.MonitorTimestamp = metav1.NewMicroTime(monitorTimestamp.Time.Add(time.Second)) + keyWithDifferentMonitorTimestamp, _, differentMonitorTimestampErr := spec.GetLifecycleKey() + require.NoError(t, differentMonitorTimestampErr) + require.NotEqual(t, keyWithMonitor, keyWithDifferentMonitorTimestamp) +} + +func TestExecutableValidateMonitorFields(t *testing.T) { + t.Parallel() + + monitorPID := int64(12345) + negativeMonitorPID := int64(-1) + monitorTimestamp := metav1.NewMicroTime(time.Now().UTC()) + + testCases := []struct { + name string + spec ExecutableSpec + expectError bool + }{ + { + name: "valid persistent monitor", + spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + Persistent: true, + MonitorPID: &monitorPID, + MonitorTimestamp: monitorTimestamp, + }, + }, + { + name: "monitor requires persistent", + spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + MonitorPID: &monitorPID, + MonitorTimestamp: monitorTimestamp, + }, + expectError: true, + }, + { + name: "monitor pid requires timestamp", + spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + Persistent: true, + MonitorPID: &monitorPID, + }, + expectError: true, + }, + { + name: "monitor timestamp requires pid", + spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + Persistent: true, + MonitorTimestamp: monitorTimestamp, + }, + expectError: true, + }, + { + name: "monitor pid must be positive", + spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + Persistent: true, + MonitorPID: &negativeMonitorPID, + MonitorTimestamp: monitorTimestamp, + }, + expectError: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + exe := &Executable{Spec: testCase.spec} + errors := exe.Validate(context.Background()) + + if testCase.expectError { + require.NotEmpty(t, errors) + } else { + require.Empty(t, errors) + } + }) + } +} + +func TestExecutableGetLifecycleKeyIgnoresImplicitEffectiveEnv(t *testing.T) { + t.Parallel() + + exe := lifecycleKeyTestExecutable() + exe.Status.EffectiveArgs = []string{"--port", "5000"} + exe.Status.EffectiveEnv = []EnvVar{ + {Name: "PATH", Value: "/first/path"}, + {Name: "EXPLICIT", Value: "stable"}, + {Name: "ASPNETCORE_URLS", Value: "http://127.0.0.1:5000"}, + } + + key, computed, keyErr := exe.GetLifecycleKey() + require.NoError(t, keyErr) + require.True(t, computed) + + exeWithDifferentImplicitEnv := lifecycleKeyTestExecutable() + exeWithDifferentImplicitEnv.Status.EffectiveArgs = []string{"--port", "5000"} + exeWithDifferentImplicitEnv.Status.EffectiveEnv = []EnvVar{ + {Name: "PATH", Value: "/different/path"}, + {Name: "EXPLICIT", Value: "stable"}, + {Name: "ASPNETCORE_URLS", Value: "http://127.0.0.1:5001"}, + } + + keyWithDifferentImplicitEnv, _, differentImplicitEnvErr := exeWithDifferentImplicitEnv.GetLifecycleKey() + require.NoError(t, differentImplicitEnvErr) + require.Equal(t, key, keyWithDifferentImplicitEnv) +} + +func TestExecutableGetLifecycleKeyIncludesExplicitEffectiveEnv(t *testing.T) { + t.Parallel() + + exe := lifecycleKeyTestExecutable() + exe.Status.EffectiveArgs = []string{"--port", "5000"} + exe.Status.EffectiveEnv = []EnvVar{ + {Name: "EXPLICIT", Value: "first"}, + } + + key, _, keyErr := exe.GetLifecycleKey() + require.NoError(t, keyErr) + + exeWithDifferentExplicitEnv := lifecycleKeyTestExecutable() + exeWithDifferentExplicitEnv.Status.EffectiveArgs = []string{"--port", "5000"} + exeWithDifferentExplicitEnv.Status.EffectiveEnv = []EnvVar{ + {Name: "EXPLICIT", Value: "second"}, + } + + keyWithDifferentExplicitEnv, _, differentExplicitEnvErr := exeWithDifferentExplicitEnv.GetLifecycleKey() + require.NoError(t, differentExplicitEnvErr) + require.NotEqual(t, key, keyWithDifferentExplicitEnv) +} + +func TestExecutableGetLifecycleKeyIncludesEffectiveArgs(t *testing.T) { + t.Parallel() + + exe := lifecycleKeyTestExecutable() + exe.Status.EffectiveArgs = []string{"--port", "5000"} + exe.Status.EffectiveEnv = []EnvVar{{Name: "EXPLICIT", Value: "stable"}} + + key, _, keyErr := exe.GetLifecycleKey() + require.NoError(t, keyErr) + + exeWithDifferentEffectiveArgs := lifecycleKeyTestExecutable() + exeWithDifferentEffectiveArgs.Status.EffectiveArgs = []string{"--port", "5001"} + exeWithDifferentEffectiveArgs.Status.EffectiveEnv = []EnvVar{{Name: "EXPLICIT", Value: "stable"}} + + keyWithDifferentEffectiveArgs, _, differentEffectiveArgsErr := exeWithDifferentEffectiveArgs.GetLifecycleKey() + require.NoError(t, differentEffectiveArgsErr) + require.NotEqual(t, key, keyWithDifferentEffectiveArgs) +} + +func TestExecutableGetLifecycleKeyRequiresEffectiveArgs(t *testing.T) { + t.Parallel() + + exe := lifecycleKeyTestExecutable() + exe.Status.EffectiveEnv = []EnvVar{{Name: "EXPLICIT", Value: "stable"}} + + _, _, keyErr := exe.GetLifecycleKey() + + require.ErrorContains(t, keyErr, "effective arguments") +} + +func TestExecutableGetLifecycleKeyRequiresEffectiveEnv(t *testing.T) { + t.Parallel() + + exe := lifecycleKeyTestExecutable() + exe.Status.EffectiveArgs = []string{"--port", "5000"} + + _, _, keyErr := exe.GetLifecycleKey() + + require.ErrorContains(t, keyErr, "effective environment") +} + +func TestExecutableGetLifecycleKeyReturnsEnvFileReadError(t *testing.T) { + t.Parallel() + + missingEnvFile := filepath.Join(t.TempDir(), "missing.env") + exe := lifecycleKeyTestExecutable() + exe.Spec.EnvFiles = []string{missingEnvFile} + exe.Status.EffectiveArgs = []string{"--port", "5000"} + exe.Status.EffectiveEnv = []EnvVar{{Name: "EXPLICIT", Value: "stable"}} + + _, _, keyErr := exe.GetLifecycleKey() + + require.ErrorContains(t, keyErr, "could not read executable environment files for lifecycle key") + require.ErrorContains(t, keyErr, missingEnvFile) +} + +func lifecycleKeyTestExecutable() *Executable { + return &Executable{ + Spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + WorkingDirectory: "/path/to/workdir", + Args: []string{"--port", "{{ port }}"}, + Env: []EnvVar{{Name: "EXPLICIT", Value: "{{ value }}"}}, + Persistent: true, + }, + } +} + +func TestExecutableSpecEqualTreatsOmittedStartAsExplicitTrue(t *testing.T) { + t.Parallel() + + shouldStart := true + shouldNotStart := false + + omittedStart := ExecutableSpec{ + ExecutablePath: "/path/to/app", + } + explicitStart := ExecutableSpec{ + ExecutablePath: "/path/to/app", + Start: &shouldStart, + } + explicitDoNotStart := ExecutableSpec{ + ExecutablePath: "/path/to/app", + Start: &shouldNotStart, + } + + assert.True(t, omittedStart.Equal(explicitStart)) + assert.False(t, omittedStart.Equal(explicitDoNotStart)) + assert.False(t, explicitStart.Equal(explicitDoNotStart)) +} + +func TestExecutableValidateUpdateStartTransitions(t *testing.T) { + t.Parallel() + + shouldStart := true + shouldNotStart := false + + testCases := []struct { + name string + oldStart *bool + newStart *bool + expectError bool + }{ + { + name: "omitted to explicit true", + oldStart: nil, + newStart: &shouldStart, + }, + { + name: "omitted to explicit false", + oldStart: nil, + newStart: &shouldNotStart, + expectError: true, + }, + { + name: "explicit true to explicit false", + oldStart: &shouldStart, + newStart: &shouldNotStart, + expectError: true, + }, + { + name: "explicit false to explicit true", + oldStart: &shouldNotStart, + newStart: &shouldStart, + }, + { + name: "explicit false to omitted", + oldStart: &shouldNotStart, + newStart: nil, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + oldExe := &Executable{ + Spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + Start: testCase.oldStart, + }, + } + newExe := &Executable{ + Spec: ExecutableSpec{ + ExecutablePath: "/path/to/app", + Start: testCase.newStart, + }, + } + + errs := newExe.ValidateUpdate(context.Background(), oldExe) + if testCase.expectError { + require.NotEmpty(t, errs) + assert.Equal(t, "spec.start", errs[0].Field) + } else { + assert.Empty(t, errs) + } + }) + } +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index b516e432..bd244e91 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,6 +1,9 @@ //go:build !ignore_autogenerated -// Copyright (c) Microsoft Corporation. All rights reserved. +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ // Code generated by controller-gen. DO NOT EDIT. @@ -705,6 +708,12 @@ func (in *ContainerSpec) DeepCopyInto(out *ContainerSpec) { } } } + if in.MonitorPID != nil { + in, out := &in.MonitorPID, &out.MonitorPID + *out = new(int64) + **out = **in + } + in.MonitorTimestamp.DeepCopyInto(&out.MonitorTimestamp) if in.RunArgs != nil { in, out := &in.RunArgs, &out.RunArgs *out = make([]string, len(*in)) @@ -1245,6 +1254,17 @@ func (in *ExecutableSpec) DeepCopyInto(out *ExecutableSpec) { copy(*out, *in) } out.AmbientEnvironment = in.AmbientEnvironment + if in.Start != nil { + in, out := &in.Start, &out.Start + *out = new(bool) + **out = **in + } + if in.MonitorPID != nil { + in, out := &in.MonitorPID, &out.MonitorPID + *out = new(int64) + **out = **in + } + in.MonitorTimestamp.DeepCopyInto(&out.MonitorTimestamp) if in.HealthProbes != nil { in, out := &in.HealthProbes, &out.HealthProbes *out = make([]HealthProbe, len(*in)) diff --git a/api/v1/zz_generated.model_name.go b/api/v1/zz_generated.model_name.go index 5b016d16..c4a8784e 100644 --- a/api/v1/zz_generated.model_name.go +++ b/api/v1/zz_generated.model_name.go @@ -1,7 +1,10 @@ //go:build !ignore_autogenerated // +build !ignore_autogenerated -// Copyright (c) Microsoft Corporation. All rights reserved. +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ // Code generated by openapi-gen. DO NOT EDIT. diff --git a/controllers/container_controller.go b/controllers/container_controller.go index b0429fbc..01c123b4 100644 --- a/controllers/container_controller.go +++ b/controllers/container_controller.go @@ -30,8 +30,10 @@ import ( apiv1 "github.com/microsoft/dcp/api/v1" "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/internal/dcpproc" "github.com/microsoft/dcp/internal/health" "github.com/microsoft/dcp/internal/networking" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/internal/templating" "github.com/microsoft/dcp/internal/version" "github.com/microsoft/dcp/pkg/commonapi" @@ -94,6 +96,9 @@ var ( type ContainerReconcilerConfig struct { MaxParallelContainerStarts uint8 ContainerStartupTimeoutOverride time.Duration + StateStore *statestore.Store + ResourceLeaseOwner process.ProcessTreeItem + ProcessExecutor process.Executor } type containerStateInitializerFunc = stateInitializerFunc[ @@ -342,6 +347,15 @@ func handleNewContainer( } if container.Spec.Persistent { + leaseErr := r.acquirePersistentContainerResourceLease(ctx, container, log) + if leaseErr != nil { + if !errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + log.Error(leaseErr, "Could not acquire persistent container resource lease") + return r.setContainerState(container, apiv1.ContainerStateFailedToStart) + } + return change | additionalReconciliationNeeded + } + // Check for an existing persistent container inspected, inspectedErr := inspectContainerIfExists(ctx, r.orchestrator, container.Spec.ContainerName) if inspectedErr != nil && !errors.Is(inspectedErr, containers.ErrNotFound) { @@ -362,6 +376,7 @@ func handleNewContainer( return change | additionalReconciliationNeeded } else { log.Error(envErr, "Could not compute effective environment for the Container") + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) return r.setContainerState(container, apiv1.ContainerStateFailedToStart) } } @@ -373,6 +388,7 @@ func handleNewContainer( return change | additionalReconciliationNeeded } else { log.Error(invocErr, "Could not compute effective invocation arguments for the Container") + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) return r.setContainerState(container, apiv1.ContainerStateFailedToStart) } } @@ -416,6 +432,7 @@ func handleNewContainer( change |= rcd.applyTo(container, log) + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) return change | r.setContainerState(container, apiv1.ContainerStateRunning) } @@ -432,6 +449,7 @@ func handleNewContainer( if !container.ShouldStart() { // We should wait to create a container until the user clears Start = false log.V(1).Info("Waiting for the container to be started") + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) return change } @@ -456,6 +474,10 @@ func ensureContainerBuildingState( if rcd == nil { // This is a brand new Container and we need to build it. + if leaseErr := r.verifyPersistentContainerResourceLeaseHeld(ctx, container, log); leaseErr != nil { + return r.setContainerState(container, apiv1.ContainerStateFailedToStart) + } + rcd = newRunningContainerData(container) rcd.containerState = apiv1.ContainerStateBuilding rcd.ensureStartupLogFiles(container, log) @@ -465,6 +487,7 @@ func ensureContainerBuildingState( err := r.startupQueue.Enqueue(r.buildImageWithOrchestrator(container, rcd.Clone(), log)) if err != nil { log.Error(err, "Image was not built, possibly because the workload is shutting down") + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) rcd.containerState = apiv1.ContainerStateFailedToStart rcd.startupError = err rcd.startAttemptFinishedAt = metav1.NowMicro() @@ -496,7 +519,8 @@ func ensureContainerStartingState( ) objectChange { change := r.setContainerState(container, apiv1.ContainerStateStarting) - if rcd == nil { + switch { + case rcd == nil: // This is brand new Container and we need to start it. rcd = newRunningContainerData(container) rcd.containerState = apiv1.ContainerStateStarting @@ -504,7 +528,7 @@ func ensureContainerStartingState( r.runningContainers.Store(container.NamespacedName(), rcd.containerID, rcd.Clone()) r.EnsureContainerWatchForResource(container.UID, log) - r.scheduleContainerCreation(container, rcd, log, startupWithNoDelay) + r.scheduleContainerCreation(ctx, container, rcd, log, startupWithNoDelay) // We need to update the runningContainers map with the new data here // because the caller did not have a runningContainerData instance. @@ -512,25 +536,25 @@ func ensureContainerStartingState( change |= statusChanged - } else if !rcd.startupAttempted { + case !rcd.startupAttempted: // We haven't attempted to start the Container yet (likely we're here after build completed). rcd.ensureStartupLogFiles(container, log) - r.scheduleContainerCreation(container, rcd, log, startupWithNoDelay) + r.scheduleContainerCreation(ctx, container, rcd, log, startupWithNoDelay) change |= statusChanged - } else if templating.IsTransientTemplateError(rcd.startupError) { - // Retry startup after transient error. + case templating.IsTransientTemplateError(rcd.startupError), errors.Is(rcd.startupError, statestore.ErrResourceLeaseHeld): + // Retry startup after a transient error or while another DCP instance is creating the persistent container. rcd.startupError = nil rcd.startAttemptFinishedAt = metav1.MicroTime{} rcd.containerName = "" - r.scheduleContainerCreation(container, rcd, log, startupRetryDelay) + r.scheduleContainerCreation(ctx, container, rcd, log, startupRetryDelay) change |= statusChanged - } else if container.Spec.Networks != nil && rcd.hasValidContainerID() { + case container.Spec.Networks != nil && rcd.hasValidContainerID(): // The second portion of startup sequence of a container with custom networks. // Need to create ContainerNetworkConnection objects and start the container resource. @@ -847,6 +871,7 @@ func (r *ContainerReconciler) removeExistingContainer( // Schedules creation of a container resource. If Container is persistent, it will attempt to find and reuse an existing container. func (r *ContainerReconciler) scheduleContainerCreation( + ctx context.Context, container *apiv1.Container, rcd *runningContainerData, log logr.Logger, @@ -856,12 +881,20 @@ func (r *ContainerReconciler) scheduleContainerCreation( rcd.startupAttempted = true + if leaseErr := r.verifyPersistentContainerResourceLeaseHeld(ctx, container, log); leaseErr != nil { + rcd.startupError = leaseErr + rcd.startAttemptFinishedAt = metav1.NowMicro() + rcd.containerState = apiv1.ContainerStateFailedToStart + return + } + log.V(1).Info("Scheduling container start", "Image", container.SpecifiedImageNameOrDefault()) if containerName == "" { uniqueContainerName, _, err := MakeUniqueName(container.Name) if err != nil { log.Error(err, "Could not generate a unique container name") + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) rcd.containerState = apiv1.ContainerStateFailedToStart rcd.startupError = err rcd.startAttemptFinishedAt = metav1.NowMicro() @@ -876,6 +909,7 @@ func (r *ContainerReconciler) scheduleContainerCreation( err := r.startupQueue.Enqueue(r.startContainerWithOrchestrator(container, rcd.Clone(), containerName, log, delay)) if err != nil { log.Error(err, "Container was not started, probably because the workload is shutting down") + _ = r.releasePersistentContainerResourceLease(ctx, container, log, false) rcd.containerState = apiv1.ContainerStateFailedToStart rcd.startupError = err rcd.startAttemptFinishedAt = metav1.NowMicro() @@ -888,6 +922,10 @@ func (r *ContainerReconciler) scheduleContainerCreation( func (r *ContainerReconciler) buildImageWithOrchestrator(container *apiv1.Container, rcd *runningContainerData, log logr.Logger) func(context.Context) { return func(buildCtx context.Context) { err := func() error { + if leaseErr := r.verifyPersistentContainerResourceLeaseHeld(buildCtx, container, log); leaseErr != nil { + return leaseErr + } + log.V(1).Info("Building image", "Dockerfile", container.Spec.Build.Dockerfile, "Context", container.Spec.Build.Context) rcd.runSpec.Build.Tags = append(rcd.runSpec.Build.Tags, container.SpecifiedImageNameOrDefault()) @@ -922,6 +960,7 @@ func (r *ContainerReconciler) buildImageWithOrchestrator(container *apiv1.Contai }() if err != nil { + _ = r.releasePersistentContainerResourceLease(context.WithoutCancel(buildCtx), container, log, false) rcd.startupError = err rcd.startAttemptFinishedAt = metav1.NowMicro() rcd.containerState = apiv1.ContainerStateFailedToStart @@ -1114,208 +1153,44 @@ func (r *ContainerReconciler) startContainerWithOrchestrator(container *apiv1.Co placeholderContainerID := rcd.containerID err := func() error { - err := r.computeEffectiveEnvironment(startupCtx, container, rcd, log) - if err != nil { - if templating.IsTransientTemplateError(err) { - log.Info("Could not compute effective environment for the Container, retrying startup...", "Cause", err.Error()) - } else { - log.Error(err, "Could not compute effective environment for the Container") - } - - return err + if leaseErr := r.verifyPersistentContainerResourceLeaseHeld(startupCtx, container, log); leaseErr != nil { + return leaseErr } - err = r.computeEffectiveInvocationArgs(startupCtx, container, rcd, log) + // Compute effective environment and invocation arguments for the container + err := r.computeContainerStartupInputs(startupCtx, container, rcd, log) if err != nil { - if templating.IsTransientTemplateError(err) { - log.Info("Could not compute effective invocation arguments for the Container, retrying startup...", "Cause", err.Error()) - } else { - log.Error(err, "Could not compute effective invocation arguments for the Container") - } - return err } - for _, volume := range container.Spec.VolumeMounts { - if volume.Type == apiv1.BindMount { - isValid := filepath.IsAbs(volume.Source) - if runtime.GOOS == "windows" { - isValid = isValid && !winNamedPipeRegex.MatchString(volume.Source) - } - if !isValid { - // This seems to be an invalid bind mount or a named pipe, so don't try to create it. - // May be a reference to a linux mount point on Windows. - log.Info("Skipping creation of bind mount because the source path appears invalid on this platform", "Source", volume.Source, "Target", volume.Target) - continue - } - - _, volErr := os.Stat(volume.Source) - if errors.Is(volErr, os.ErrNotExist) { - volErr = os.MkdirAll(volume.Source, osutil.PermissionDirectoryOthersRead) - if volErr != nil { - log.Error(volErr, "Could not create bind mount source path", "Source", volume.Source, "Target", volume.Target) - return volErr - } - } else if volErr != nil { - log.Error(volErr, "Could not verify existence of bind mount source path", "Volume", volume.Source, "Target", volume.Target) - return volErr - } - } + // Process any bind mounts to ensure there are valid sources on the host + err = ensureContainerBindMountSources(container, log) + if err != nil { + return err } log.V(1).Info("Starting container", "Image", container.SpecifiedImageNameOrDefault()) + // Determine the default bridge network name used by the active orchestrator defaultNetwork := "" if rcd.runSpec.Networks != nil { // See comment below why we create the container with default network explicitly enabled here. defaultNetwork = r.orchestrator.DefaultNetworkName() } - lifecycleKey, _, hashErr := rcd.runSpec.GetLifecycleKey() + // Add labels to the container for lifecycle management and other metadata + lifecycleKey, hashErr := r.addContainerCreationLabels(container, rcd, log) if hashErr != nil { - log.Error(hashErr, "Could not compute lifecycle key for the container") return hashErr } - rcd.runSpec.Labels = append(rcd.runSpec.Labels, []apiv1.ContainerLabel{ - { - Key: dcpBuildLabel, - Value: version.ProductVersion, - }, - { - Key: groupVersionLabel, - Value: apiv1.GroupVersion.String(), - }, - { - Key: uidLabel, - Value: string(container.UID), - }, - { - Key: nameLabel, - Value: string(container.Name), - }, - { - Key: lifecycleKeyLabel, - Value: lifecycleKey, - }, - { - Key: PersistentLabel, - Value: fmt.Sprintf("%t", rcd.runSpec.Persistent), - }, - }...) - - thisProcess, thisProcessErr := process.This() - if thisProcessErr != nil { - log.Error(thisProcessErr, "Could not get the current process information; container will not have creator process information") - } else { - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: CreatorProcessIdLabel, - Value: fmt.Sprintf("%d", thisProcess.Pid), - }) - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: CreatorProcessStartTimeLabel, - Value: thisProcess.IdentityTime.Format(osutil.RFC3339MiliTimestampFormat), - }) - } - - if len(rcd.runSpec.Env) > 0 { - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: envLabel, - Value: strings.Join(slices.Map[string](rcd.runSpec.Env, func(env apiv1.EnvVar) string { - return env.Name - }), "\n"), - }) - } - - if len(rcd.runSpec.Ports) > 0 { - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: portsLabel, - Value: strings.Join(slices.Map[string](rcd.runSpec.Ports, func(port apiv1.ContainerPort) string { - protocol := "tcp" - if port.Protocol != "" { - protocol = strings.ToLower(string(port.Protocol)) - } - return fmt.Sprintf("%d/%s", port.ContainerPort, protocol) - }), "\n"), - }) - } - - if len(rcd.runSpec.VolumeMounts) > 0 { - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: mountsLabel, - Value: strings.Join(slices.Map[string](rcd.runSpec.VolumeMounts, func(mount apiv1.VolumeMount) string { - return fmt.Sprintf("type=%s,src=%s", mount.Type, mount.Source) - }), "\n"), - }) - } - - if len(rcd.runSpec.CreateFiles) > 0 { - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: createFilesLabel, - Value: fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%v", rcd.runSpec.CreateFiles)))), - }) - } - - if rcd.runSpec.PemCertificates != nil { - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: pemCertificatesLabel, - Value: fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%v", rcd.runSpec.PemCertificates)))), - }) - } - - // If image layers are specified, build a derived image with the layers applied - effectiveImage := rcd.runSpec.Image - if len(rcd.runSpec.ImageLayers) > 0 { - digests := make([]string, len(rcd.runSpec.ImageLayers)) - for i, layer := range rcd.runSpec.ImageLayers { - digests[i] = layer.Digest - } - rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ - Key: imageLayersLabel, - Value: fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(digests, "\n")))), - }) - - // When applying image layers, we need the base image locally before - // constructing the derived image. Normally docker create --pull handles - // pulling, but since we intercept before that, we must respect PullPolicy here. - baseImageInspected, ensureErr := ensureBaseImageForLayers( - startupCtx, r.orchestrator, rcd.runSpec.Image, rcd.runSpec.PullPolicy, log, - ) - if ensureErr != nil { - log.Error(ensureErr, "Could not ensure base image is available for image layers") - return ensureErr - } - - // Derive a tag by replacing the base image's tag/digest with the lifecycle key. - // The image ref may be name:tag or name@sha256:digest — we need just the name part. - baseRepo := rcd.runSpec.Image - if atidx := strings.Index(baseRepo, "@"); atidx != -1 { - baseRepo = baseRepo[:atidx] - } else if colonidx := strings.LastIndex(baseRepo, ":"); colonidx != -1 { - // Only strip at colon if it's a tag separator, not part of a port/registry. - // A tag separator colon comes after the last slash (or is the only colon). - slashIdx := strings.LastIndex(baseRepo, "/") - if colonidx > slashIdx { - baseRepo = baseRepo[:colonidx] - } - } - sanitizedKey := strings.ReplaceAll(lifecycleKey, ":", "-") - derivedTag := fmt.Sprintf("%s:dcp-%s", baseRepo, sanitizedKey) - applyLayersOptions := containers.ApplyImageLayersOptions{ - BaseImage: *baseImageInspected, - Layers: rcd.runSpec.ImageLayers, - Tag: derivedTag, - } - derivedImage, applyErr := r.orchestrator.ApplyImageLayers(startupCtx, applyLayersOptions) - if applyErr != nil { - log.Error(applyErr, "Could not apply image layers") - return applyErr - } - - log.V(1).Info("Applied image layers to create derived image", "DerivedImage", derivedImage) - effectiveImage = derivedImage + // Apply any image layers that are specified in the container spec, and get the effective image to use for container creation + effectiveImage, imageLayersErr := r.applyContainerImageLayers(startupCtx, rcd, lifecycleKey, log) + if imageLayersErr != nil { + return imageLayersErr } + // Get the writers for the startup log files and prepare the stream options for container creation startupStdoutWriter, startupStderrWriter := rcd.getStartupLogWriters() streamOptions := getStartupCommandOptions(startupStdoutWriter, startupStderrWriter) @@ -1327,17 +1202,22 @@ func (r *ContainerReconciler) startContainerWithOrchestrator(container *apiv1.Co // attempting to pull it from a registry. runSpecForCreation.PullPolicy = apiv1.PullPolicyNever } + + // Create the container using the active orchestrator. + // Persistent container reuse happens only in handleNewContainer before startup is scheduled. creationOptions := containers.CreateContainerOptions{ ContainerSpec: runSpecForCreation, Name: containerName, Network: defaultNetwork, StreamCommandOptions: streamOptions, } - inspected, err := createContainer(startupCtx, r.orchestrator, creationOptions) + inspected, createErr := createContainer(startupCtx, r.orchestrator, creationOptions) + + // Close the startup log writers to ensure all output is flushed and resources are released startupTaskFinished(startupStdoutWriter, startupStderrWriter) - if err != nil { - log.Error(err, "Could not create the container") - return err + if createErr != nil { + log.Error(createErr, "Could not create the container") + return createErr } log = log.WithValues("ContainerID", GetShortId(inspected.Id)) @@ -1345,174 +1225,40 @@ func (r *ContainerReconciler) startContainerWithOrchestrator(container *apiv1.Co log.V(1).Info("Container created") rcd.updateFromInspectedContainer(inspected) + // Copy any general files specified in the container spec to the container's filesystem fileModTime := time.Now() - for _, createFileRequest := range rcd.runSpec.CreateFiles { - umask := osutil.DefaultUmaskBitmask - if createFileRequest.Umask != nil { - umask = *createFileRequest.Umask - } - - createFilesOptions := containers.CreateFilesOptions{ - Container: inspected.Id, - Entries: createFileRequest.Entries, - Destination: createFileRequest.Destination, - DefaultOwner: createFileRequest.DefaultOwner, - DefaultGroup: createFileRequest.DefaultGroup, - Umask: umask, - ModTime: fileModTime, - } - - copyErr := r.orchestrator.CreateFiles(startupCtx, createFilesOptions) - if copyErr != nil { - log.Error(copyErr, "Could not copy files to the container", "Destination", createFileRequest.Destination) - return copyErr - } - - log.V(1).Info("Files copied to the container", "Destination", createFileRequest.Destination) + copyFilesErr := r.copyContainerCreateFiles(startupCtx, rcd, inspected, fileModTime, log) + if copyFilesErr != nil { + return copyFilesErr } - if rcd.runSpec.PemCertificates != nil { - fingerprints := []string{} - certFiles := []apiv1.FileSystemEntry{} - bundle := []string{} - for _, cert := range rcd.runSpec.PemCertificates.Certificates { - fingerprint, fingerprintErr := cert.OpenSSLFingerprint() - if fingerprintErr != nil { - if rcd.runSpec.PemCertificates.ContinueOnError { - log.Info("Could not compute certificate fingerprint, skipping certificate", "Thumbprint", cert.Thumbprint, "Error", fingerprintErr.Error()) - continue - } - - return fingerprintErr - } - - collisions := 0 - for _, existingFingerprint := range fingerprints { - if existingFingerprint == fingerprint { - collisions++ - } - } - - fingerprints = append(fingerprints, fingerprint) - - certFiles = append( - certFiles, - apiv1.FileSystemEntry{ - Name: fmt.Sprintf("%s.pem", cert.Thumbprint), - Contents: cert.Contents, - ContinueOnError: rcd.runSpec.PemCertificates.ContinueOnError, - }, - apiv1.FileSystemEntry{ - Name: fmt.Sprintf("%s.%d", fingerprint, collisions), - Type: apiv1.FileSystemEntryTypeSymlink, - Target: fmt.Sprintf("./%s.pem", cert.Thumbprint), - ContinueOnError: rcd.runSpec.PemCertificates.ContinueOnError, - }) - - bundle = append(bundle, cert.Contents) - } - - createFilesOptions := containers.CreateFilesOptions{ - Container: inspected.Id, - Destination: rcd.runSpec.PemCertificates.Destination, - Entries: []apiv1.FileSystemEntry{ - { - Name: "cert.pem", - Contents: strings.Join(bundle, "\n"), - }, - { - Name: "certs", - Type: apiv1.FileSystemEntryTypeDir, - Entries: certFiles, - }, - }, - Umask: osutil.DefaultUmaskBitmask, - ModTime: fileModTime, - } - - copyErr := r.orchestrator.CreateFiles(startupCtx, createFilesOptions) - if copyErr != nil { - log.Error(copyErr, "Could not copy certificates to the container", "Destination", createFilesOptions.Destination) - return copyErr - } - - overwritePaths := slices.GroupBy[[]string, string](rcd.runSpec.PemCertificates.OverwriteBundlePaths, func(bundlePath string) string { - return path.Dir(bundlePath) - }) - - for bundleDir, files := range overwritePaths { - bundleCreateFilesOptions := containers.CreateFilesOptions{ - Container: inspected.Id, - Destination: bundleDir, - Entries: slices.Map[apiv1.FileSystemEntry](files, func(file string) apiv1.FileSystemEntry { - return apiv1.FileSystemEntry{ - Name: path.Base(file), - Contents: strings.Join(bundle, "\n"), - ContinueOnError: rcd.runSpec.PemCertificates.ContinueOnError, - } - }), - Umask: osutil.DefaultUmaskBitmask, - ModTime: fileModTime, - } - - bundleCopyErr := r.orchestrator.CreateFiles(startupCtx, bundleCreateFilesOptions) - if bundleCopyErr != nil { - if rcd.runSpec.PemCertificates.ContinueOnError { - log.Info("Could not overwrite certificate bundle in the container, continuing...", "Destination", bundleDir, "Error", bundleCopyErr.Error()) - } else { - log.Error(bundleCopyErr, "Could not overwrite certificate bundle in the container", "Destination", bundleDir) - return bundleCopyErr - } - } - } - - log.V(1).Info("Certificates copied to the container", "Destination", createFilesOptions.Destination) + // Copy any PEM certificates specified in the container spec to the container's filesystem + copyCertsErr := r.copyContainerPemCertificates(startupCtx, rcd, inspected, fileModTime, log) + if copyCertsErr != nil { + return copyCertsErr } - if rcd.runSpec.Networks == nil { - inspected, err = r.startContainerWithTimeout(startupCtx, containerName, rcd.containerID, streamOptions) - rcd.startAttemptFinishedAt = metav1.NowMicro() - startupTaskFinished(startupStdoutWriter, startupStderrWriter) - if err != nil { - log.Error(err, "Could not start the container") - return err - } - - if inspected.Status == containers.ContainerStatusRunning { - log.V(1).Info("Container started") - rcd.containerState = apiv1.ContainerStateRunning - } else { - log.V(1).Info("Container started and exited shortly after", "ContainerStatus", inspected.Status) - rcd.containerState = apiv1.ContainerStateExited - } - } else { - // If a container resource is created without a network, it cannot be connected to a network later (orchestrator limitation). - // So for Containers that request attaching to custom networks via Spec, we create the corresponding container resource - // attached to default network(s) (usually one: "bridge" for Docker or "podman" for Podman). - // Here we detach it from the default network(s). Then we leave the Container object in "starting" state, - // and save the changes. - // - // During next reconciliation loop we create ContainerNetworkConnection objects and start the container resource. - // The Network controller takes care of connecting the container resource to requested networks. - for i := range inspected.Networks { - networkID := inspected.Networks[i].Id - err = disconnectNetwork(startupCtx, r.orchestrator, containers.DisconnectNetworkOptions{Network: networkID, Container: string(rcd.containerID), Force: true}) - if err != nil { - log.Error(err, "Could not detach network from the container", "NetworkID", networkID) - return err - } - } + // Finish the container startup process, including any finalization steps + finishErr := r.finishCreatedContainerStartup(startupCtx, containerName, rcd, inspected, streamOptions, startupStdoutWriter, startupStderrWriter, log) + if finishErr != nil { + return finishErr } return nil }() + retryableErr := templating.IsTransientTemplateError(err) || errors.Is(err, statestore.ErrResourceLeaseHeld) + if !retryableErr && !errors.Is(err, statestore.ErrResourceLeaseNotHeld) { + releaseErr := r.releasePersistentContainerResourceLease(context.WithoutCancel(startupCtx), container, log, false) + err = errors.Join(err, releaseErr) + } + if err != nil { rcd.startupError = err rcd.startAttemptFinishedAt = metav1.NowMicro() - // Keep in "starting" state if the error is a transient error, otherwise initiate the transition to "failed to start". - if !templating.IsTransientTemplateError(err) { + // Keep in "starting" state if the error is transient, otherwise initiate the transition to "failed to start". + if !retryableErr { rcd.containerState = apiv1.ContainerStateFailedToStart } } @@ -1525,6 +1271,490 @@ func (r *ContainerReconciler) startContainerWithOrchestrator(container *apiv1.Co } } +// computeContainerStartupInputs resolves environment variables and invocation args before container creation. +func (r *ContainerReconciler) computeContainerStartupInputs(ctx context.Context, container *apiv1.Container, rcd *runningContainerData, log logr.Logger) error { + envErr := r.computeEffectiveEnvironment(ctx, container, rcd, log) + if envErr != nil { + if templating.IsTransientTemplateError(envErr) { + log.Info("Could not compute effective environment for the Container, retrying startup...", "Cause", envErr.Error()) + } else { + log.Error(envErr, "Could not compute effective environment for the Container") + } + + return envErr + } + + invocErr := r.computeEffectiveInvocationArgs(ctx, container, rcd, log) + if invocErr != nil { + if templating.IsTransientTemplateError(invocErr) { + log.Info("Could not compute effective invocation arguments for the Container, retrying startup...", "Cause", invocErr.Error()) + } else { + log.Error(invocErr, "Could not compute effective invocation arguments for the Container") + } + + return invocErr + } + + return nil +} + +// ensureContainerBindMountSources creates missing host paths for valid bind-mount sources. +func ensureContainerBindMountSources(container *apiv1.Container, log logr.Logger) error { + for _, volume := range container.Spec.VolumeMounts { + if volume.Type != apiv1.BindMount { + continue + } + + isValid := filepath.IsAbs(volume.Source) + if runtime.GOOS == "windows" { + isValid = isValid && !winNamedPipeRegex.MatchString(volume.Source) + } + if !isValid { + // This seems to be an invalid bind mount or a named pipe, so don't try to create it. + // May be a reference to a linux mount point on Windows. + log.Info("Skipping creation of bind mount because the source path appears invalid on this platform", "Source", volume.Source, "Target", volume.Target) + continue + } + + _, volErr := os.Stat(volume.Source) + if errors.Is(volErr, os.ErrNotExist) { + mkdirErr := os.MkdirAll(volume.Source, osutil.PermissionDirectoryOthersRead) + if mkdirErr != nil { + log.Error(mkdirErr, "Could not create bind mount source path", "Source", volume.Source, "Target", volume.Target) + return mkdirErr + } + } else if volErr != nil { + log.Error(volErr, "Could not verify existence of bind mount source path", "Volume", volume.Source, "Target", volume.Target) + return volErr + } + } + + return nil +} + +// addContainerCreationLabels adds lifecycle and diagnostic labels used to manage or inspect the container later. +func (r *ContainerReconciler) addContainerCreationLabels(container *apiv1.Container, rcd *runningContainerData, log logr.Logger) (string, error) { + lifecycleKey, _, hashErr := rcd.runSpec.GetLifecycleKey() + if hashErr != nil { + log.Error(hashErr, "Could not compute lifecycle key for the container") + return "", hashErr + } + + rcd.runSpec.Labels = append(rcd.runSpec.Labels, []apiv1.ContainerLabel{ + { + Key: dcpBuildLabel, + Value: version.ProductVersion, + }, + { + Key: groupVersionLabel, + Value: apiv1.GroupVersion.String(), + }, + { + Key: uidLabel, + Value: string(container.UID), + }, + { + Key: nameLabel, + Value: string(container.Name), + }, + { + Key: lifecycleKeyLabel, + Value: lifecycleKey, + }, + { + Key: PersistentLabel, + Value: fmt.Sprintf("%t", rcd.runSpec.Persistent), + }, + }...) + + thisProcess, thisProcessErr := process.This() + if thisProcessErr != nil { + log.Error(thisProcessErr, "Could not get the current process information; container will not have creator process information") + } else { + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: CreatorProcessIdLabel, + Value: fmt.Sprintf("%d", thisProcess.Pid), + }) + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: CreatorProcessStartTimeLabel, + Value: thisProcess.IdentityTime.Format(osutil.RFC3339MiliTimestampFormat), + }) + } + + if len(rcd.runSpec.Env) > 0 { + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: envLabel, + Value: strings.Join(slices.Map[string](rcd.runSpec.Env, func(env apiv1.EnvVar) string { + return env.Name + }), "\n"), + }) + } + + if len(rcd.runSpec.Ports) > 0 { + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: portsLabel, + Value: strings.Join(slices.Map[string](rcd.runSpec.Ports, func(port apiv1.ContainerPort) string { + protocol := "tcp" + if port.Protocol != "" { + protocol = strings.ToLower(string(port.Protocol)) + } + return fmt.Sprintf("%d/%s", port.ContainerPort, protocol) + }), "\n"), + }) + } + + if len(rcd.runSpec.VolumeMounts) > 0 { + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: mountsLabel, + Value: strings.Join(slices.Map[string](rcd.runSpec.VolumeMounts, func(mount apiv1.VolumeMount) string { + return fmt.Sprintf("type=%s,src=%s", mount.Type, mount.Source) + }), "\n"), + }) + } + + if len(rcd.runSpec.CreateFiles) > 0 { + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: createFilesLabel, + Value: fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%v", rcd.runSpec.CreateFiles)))), + }) + } + + if rcd.runSpec.PemCertificates != nil { + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: pemCertificatesLabel, + Value: fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%v", rcd.runSpec.PemCertificates)))), + }) + } + + return lifecycleKey, nil +} + +// applyContainerImageLayers builds a derived image when image layers are configured and returns the image to create. +func (r *ContainerReconciler) applyContainerImageLayers(ctx context.Context, rcd *runningContainerData, lifecycleKey string, log logr.Logger) (string, error) { + effectiveImage := rcd.runSpec.Image + if len(rcd.runSpec.ImageLayers) == 0 { + return effectiveImage, nil + } + + digests := make([]string, len(rcd.runSpec.ImageLayers)) + for i, layer := range rcd.runSpec.ImageLayers { + digests[i] = layer.Digest + } + rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ + Key: imageLayersLabel, + Value: fmt.Sprintf("%x", sha256.Sum256([]byte(strings.Join(digests, "\n")))), + }) + + // When applying image layers, we need the base image locally before + // constructing the derived image. Normally docker create --pull handles + // pulling, but since we intercept before that, we must respect PullPolicy here. + baseImageInspected, ensureErr := ensureBaseImageForLayers( + ctx, r.orchestrator, rcd.runSpec.Image, rcd.runSpec.PullPolicy, log, + ) + if ensureErr != nil { + log.Error(ensureErr, "Could not ensure base image is available for image layers") + return "", ensureErr + } + + // Derive a tag by replacing the base image's tag/digest with the lifecycle key. + // The image ref may be name:tag or name@sha256:digest — we need just the name part. + baseRepo := rcd.runSpec.Image + if atidx := strings.Index(baseRepo, "@"); atidx != -1 { + baseRepo = baseRepo[:atidx] + } else if colonidx := strings.LastIndex(baseRepo, ":"); colonidx != -1 { + // Only strip at colon if it's a tag separator, not part of a port/registry. + // A tag separator colon comes after the last slash (or is the only colon). + slashIdx := strings.LastIndex(baseRepo, "/") + if colonidx > slashIdx { + baseRepo = baseRepo[:colonidx] + } + } + sanitizedKey := strings.ReplaceAll(lifecycleKey, ":", "-") + derivedTag := fmt.Sprintf("%s:dcp-%s", baseRepo, sanitizedKey) + applyLayersOptions := containers.ApplyImageLayersOptions{ + BaseImage: *baseImageInspected, + Layers: rcd.runSpec.ImageLayers, + Tag: derivedTag, + } + derivedImage, applyErr := r.orchestrator.ApplyImageLayers(ctx, applyLayersOptions) + if applyErr != nil { + log.Error(applyErr, "Could not apply image layers") + return "", applyErr + } + + log.V(1).Info("Applied image layers to create derived image", "DerivedImage", derivedImage) + return derivedImage, nil +} + +func (r *ContainerReconciler) acquirePersistentContainerResourceLease( + ctx context.Context, + container *apiv1.Container, + log logr.Logger, +) error { + if !container.Spec.Persistent { + return nil + } + if r.config.StateStore == nil { + return fmt.Errorf("state store is not configured") + } + + lease, leaseErr := r.config.StateStore.AcquireResourceLease( + ctx, + container, + r.config.ResourceLeaseOwner, + resourceLeaseRevalidationInterval, + ) + if errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + logResourceLeaseHeld(log, leaseErr, container.GetLeaseKey(), "Persistent container is being updated by another DCP instance, retrying") + } + if leaseErr != nil { + return leaseErr + } + + log.V(1).Info("Acquired resource lease", "ResourceKey", lease.ResourceKey) + return nil +} + +func (r *ContainerReconciler) verifyPersistentContainerResourceLeaseHeld(ctx context.Context, container *apiv1.Container, log logr.Logger) error { + if !container.Spec.Persistent { + return nil + } + if r.config.StateStore == nil { + return fmt.Errorf("state store is not configured") + } + + leaseErr := r.config.StateStore.VerifyResourceLeaseHeld(ctx, container, r.config.ResourceLeaseOwner) + if leaseErr != nil { + log.Error(leaseErr, "Cannot continue persistent container startup because this DCP instance does not hold the resource lease") + return leaseErr + } + + return nil +} + +func (r *ContainerReconciler) releasePersistentContainerResourceLease( + ctx context.Context, + container *apiv1.Container, + log logr.Logger, + suppressNotHeldLog bool, +) error { + if !container.Spec.Persistent { + return nil + } + if r.config.StateStore == nil { + return fmt.Errorf("state store is not configured") + } + + releaseErr := r.config.StateStore.ReleaseResourceLease(ctx, container, r.config.ResourceLeaseOwner) + if releaseErr != nil { + if errors.Is(releaseErr, statestore.ErrResourceLeaseNotHeld) { + logResourceLeaseNotHeld(log, suppressNotHeldLog, container.GetLeaseKey(), "Persistent container resource lease was not held") + return releaseErr + } + log.Error(releaseErr, "Could not release persistent container resource lease") + return releaseErr + } + + return nil +} + +// copyContainerCreateFiles copies configured file entries into the created container before it starts. +func (r *ContainerReconciler) copyContainerCreateFiles( + ctx context.Context, + rcd *runningContainerData, + inspected *containers.InspectedContainer, + fileModTime time.Time, + log logr.Logger, +) error { + for _, createFileRequest := range rcd.runSpec.CreateFiles { + umask := osutil.DefaultUmaskBitmask + if createFileRequest.Umask != nil { + umask = *createFileRequest.Umask + } + + createFilesOptions := containers.CreateFilesOptions{ + Container: inspected.Id, + Entries: createFileRequest.Entries, + Destination: createFileRequest.Destination, + DefaultOwner: createFileRequest.DefaultOwner, + DefaultGroup: createFileRequest.DefaultGroup, + Umask: umask, + ModTime: fileModTime, + } + + copyErr := r.orchestrator.CreateFiles(ctx, createFilesOptions) + if copyErr != nil { + log.Error(copyErr, "Could not copy files to the container", "Destination", createFileRequest.Destination) + return copyErr + } + + log.V(1).Info("Files copied to the container", "Destination", createFileRequest.Destination) + } + + return nil +} + +// copyContainerPemCertificates installs configured PEM certificates and optional bundle overwrites into the container. +func (r *ContainerReconciler) copyContainerPemCertificates( + ctx context.Context, + rcd *runningContainerData, + inspected *containers.InspectedContainer, + fileModTime time.Time, + log logr.Logger, +) error { + if rcd.runSpec.PemCertificates == nil { + return nil + } + + fingerprints := []string{} + certFiles := []apiv1.FileSystemEntry{} + bundle := []string{} + for _, cert := range rcd.runSpec.PemCertificates.Certificates { + fingerprint, fingerprintErr := cert.OpenSSLFingerprint() + if fingerprintErr != nil { + if rcd.runSpec.PemCertificates.ContinueOnError { + log.Info("Could not compute certificate fingerprint, skipping certificate", "Thumbprint", cert.Thumbprint, "Error", fingerprintErr.Error()) + continue + } + + return fingerprintErr + } + + collisions := 0 + for _, existingFingerprint := range fingerprints { + if existingFingerprint == fingerprint { + collisions++ + } + } + + fingerprints = append(fingerprints, fingerprint) + + certFiles = append( + certFiles, + apiv1.FileSystemEntry{ + Name: fmt.Sprintf("%s.pem", cert.Thumbprint), + Contents: cert.Contents, + ContinueOnError: rcd.runSpec.PemCertificates.ContinueOnError, + }, + apiv1.FileSystemEntry{ + Name: fmt.Sprintf("%s.%d", fingerprint, collisions), + Type: apiv1.FileSystemEntryTypeSymlink, + Target: fmt.Sprintf("./%s.pem", cert.Thumbprint), + ContinueOnError: rcd.runSpec.PemCertificates.ContinueOnError, + }) + + bundle = append(bundle, cert.Contents) + } + + createFilesOptions := containers.CreateFilesOptions{ + Container: inspected.Id, + Destination: rcd.runSpec.PemCertificates.Destination, + Entries: []apiv1.FileSystemEntry{ + { + Name: "cert.pem", + Contents: strings.Join(bundle, "\n"), + }, + { + Name: "certs", + Type: apiv1.FileSystemEntryTypeDir, + Entries: certFiles, + }, + }, + Umask: osutil.DefaultUmaskBitmask, + ModTime: fileModTime, + } + + copyErr := r.orchestrator.CreateFiles(ctx, createFilesOptions) + if copyErr != nil { + log.Error(copyErr, "Could not copy certificates to the container", "Destination", createFilesOptions.Destination) + return copyErr + } + + overwritePaths := slices.GroupBy[[]string, string](rcd.runSpec.PemCertificates.OverwriteBundlePaths, func(bundlePath string) string { + return path.Dir(bundlePath) + }) + + for bundleDir, files := range overwritePaths { + bundleCreateFilesOptions := containers.CreateFilesOptions{ + Container: inspected.Id, + Destination: bundleDir, + Entries: slices.Map[apiv1.FileSystemEntry](files, func(file string) apiv1.FileSystemEntry { + return apiv1.FileSystemEntry{ + Name: path.Base(file), + Contents: strings.Join(bundle, "\n"), + ContinueOnError: rcd.runSpec.PemCertificates.ContinueOnError, + } + }), + Umask: osutil.DefaultUmaskBitmask, + ModTime: fileModTime, + } + + bundleCopyErr := r.orchestrator.CreateFiles(ctx, bundleCreateFilesOptions) + if bundleCopyErr != nil { + if rcd.runSpec.PemCertificates.ContinueOnError { + log.Info("Could not overwrite certificate bundle in the container, continuing...", "Destination", bundleDir, "Error", bundleCopyErr.Error()) + } else { + log.Error(bundleCopyErr, "Could not overwrite certificate bundle in the container", "Destination", bundleDir) + return bundleCopyErr + } + } + } + + log.V(1).Info("Certificates copied to the container", "Destination", createFilesOptions.Destination) + return nil +} + +// finishCreatedContainerStartup either starts the container immediately or prepares it for custom network attachment. +func (r *ContainerReconciler) finishCreatedContainerStartup( + ctx context.Context, + containerName string, + rcd *runningContainerData, + inspected *containers.InspectedContainer, + streamOptions containers.StreamCommandOptions, + startupStdoutWriter usvc_io.ParagraphWriter, + startupStderrWriter usvc_io.ParagraphWriter, + log logr.Logger, +) error { + if rcd.runSpec.Networks == nil { + startedContainer, startErr := r.startContainerWithTimeout(ctx, containerName, rcd.containerID, streamOptions) + rcd.startAttemptFinishedAt = metav1.NowMicro() + startupTaskFinished(startupStdoutWriter, startupStderrWriter) + if startErr != nil { + log.Error(startErr, "Could not start the container") + return startErr + } + + if startedContainer.Status == containers.ContainerStatusRunning { + log.V(1).Info("Container started") + rcd.containerState = apiv1.ContainerStateRunning + } else { + log.V(1).Info("Container started and exited shortly after", "ContainerStatus", startedContainer.Status) + rcd.containerState = apiv1.ContainerStateExited + } + r.runPersistentContainerLifecycleMonitor(rcd, log) + return nil + } + + // If a container resource is created without a network, it cannot be connected to a network later (orchestrator limitation). + // So for Containers that request attaching to custom networks via Spec, we create the corresponding container resource + // attached to default network(s) (usually one: "bridge" for Docker or "podman" for Podman). + // Here we detach it from the default network(s). Then we leave the Container object in "starting" state, + // and save the changes. + // + // During next reconciliation loop we create ContainerNetworkConnection objects and start the container resource. + // The Network controller takes care of connecting the container resource to requested networks. + for i := range inspected.Networks { + networkID := inspected.Networks[i].Id + disconnectErr := disconnectNetwork(ctx, r.orchestrator, containers.DisconnectNetworkOptions{Network: networkID, Container: string(rcd.containerID), Force: true}) + if disconnectErr != nil { + log.Error(disconnectErr, "Could not detach network from the container", "NetworkID", networkID) + return disconnectErr + } + } + + return nil +} + // Returns a function that stops the container resource by calling the container orchestrator. // The method is called as part of the reconciliation loop, but the returned function is executed asynchronously. // The passed runningContainerData should be a clone independent from what is stored in the runningContainers map. @@ -1675,10 +1905,32 @@ func (r *ContainerReconciler) handleInitialNetworkConnections( log.Error(startupErr, "Failed to start Container") return nil, startupErr } + r.runPersistentContainerLifecycleMonitor(rcd, log) return inspected, nil } +func (r *ContainerReconciler) runPersistentContainerLifecycleMonitor(rcd *runningContainerData, log logr.Logger) { + if rcd == nil || rcd.runSpec == nil || !rcd.runSpec.Persistent { + return + } + + monitor, found, monitorErr := dcpproc.MonitorTargetFromFields(rcd.runSpec.MonitorPID, rcd.runSpec.MonitorTimestamp) + if monitorErr != nil { + log.Error(monitorErr, "Could not start persistent Container lifecycle monitor") + return + } + if !found { + return + } + if r.config.ProcessExecutor == nil { + log.Error(fmt.Errorf("process executor is not configured"), "Could not start persistent Container lifecycle monitor") + return + } + + dcpproc.RunContainerWatcherForMonitor(r.config.ProcessExecutor, monitor, string(rcd.containerID), log) +} + // Connects the Container to networks as necessary, updating status. // This method is executed as part of the reconciliation loop. func (r *ContainerReconciler) handleContainerNetworkConnections( @@ -2052,11 +2304,30 @@ func (r *ContainerReconciler) setContainerState(container *apiv1.Container, stat change = statusChanged } + if container.Spec.Persistent && persistentContainerLeaseReleaseState(state) { + // Intentionally ignore errors: this is a defensive, idempotent release attempt for stable states. + _ = r.releasePersistentContainerResourceLease(context.Background(), container, r.Log, true) + } + change |= updateContainerHealthStatus(container, state, r.Log) return change } +func persistentContainerLeaseReleaseState(state apiv1.ContainerState) bool { + switch state { + case apiv1.ContainerStateRuntimeUnhealthy, + apiv1.ContainerStateFailedToStart, + apiv1.ContainerStateRunning, + apiv1.ContainerStatePaused, + apiv1.ContainerStateExited, + apiv1.ContainerStateUnknown: + return true + default: + return false + } +} + func updateContainerHealthStatus(ctr *apiv1.Container, state apiv1.ContainerState, log logr.Logger) objectChange { var newHealthStatus apiv1.HealthStatus diff --git a/controllers/controller_common.go b/controllers/controller_common.go index f7625e2c..d2b33a0c 100644 --- a/controllers/controller_common.go +++ b/controllers/controller_common.go @@ -37,9 +37,10 @@ const ( specChanged objectChange = 0x4 additionalReconciliationNeeded objectChange = 0x8 - conflictRequeueDelay = 100 * time.Millisecond - reconciliationDebounceDelay = 500 * time.Millisecond - reconciliationMaxDelay = 5 * time.Second + conflictRequeueDelay = 100 * time.Millisecond + reconciliationDebounceDelay = 500 * time.Millisecond + reconciliationMaxDelay = 5 * time.Second + resourceLeaseRevalidationInterval = 30 * time.Second PersistentLabel = "com.microsoft.developer.usvc-dev.persistent" CreatorProcessIdLabel = "com.microsoft.developer.usvc-dev.creatorProcessId" diff --git a/controllers/endpoint_common.go b/controllers/endpoint_common.go index b7c4f459..89463a05 100644 --- a/controllers/endpoint_common.go +++ b/controllers/endpoint_common.go @@ -3,233 +3,232 @@ * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ - -package controllers - -import ( - "context" - - "github.com/go-logr/logr" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - ctrl_client "sigs.k8s.io/controller-runtime/pkg/client" - - apiv1 "github.com/microsoft/dcp/api/v1" - "github.com/microsoft/dcp/pkg/commonapi" - "github.com/microsoft/dcp/pkg/logger" - "github.com/microsoft/dcp/pkg/slices" - "github.com/microsoft/dcp/pkg/syncmap" -) - -// The cache of endpoints created for a given workload and service combination. -var workloadEndpointCache *syncmap.Map[ServiceWorkloadEndpointKey, []apiv1.Endpoint] = &syncmap.Map[ServiceWorkloadEndpointKey, []apiv1.Endpoint]{} - -type ServiceWorkloadEndpointKey struct { - commonapi.NamespacedNameWithKind - ServiceName string -} - -type EndpointOwner[EndpointCreationContext any] interface { - ctrl_client.Client - - // Creates (but does not persist) new Endpoint objects for the given Service - // exposed by the workload object (owner parameter). The Service is represented by the serviceProducer parameter. - // Additional context data needed for Endpoint creation is passed in the ecc parameter. - // Existing Endpoints for the same workload and service combination are passed via the existingEndpoints parameter. - // The function should ONLY create new Endpoints that are "missing", i.e. the existing Endpoint is not sufficient. - createEndpoints( - ctx context.Context, - owner ctrl_client.Object, - serviceProducer commonapi.ServiceProducer, - existingEndpoints []*apiv1.Endpoint, - ecc EndpointCreationContext, - log logr.Logger, - ) ([]*apiv1.Endpoint, error) - - // Validates that the existing Endpoint objects correctly represent the Service exposed by the workload object. - // Returns two lists: the first list contains valid existing Endpoints that can be kept as-is, - // the second list contains invalid existing Endpoints that should be deleted. - validateExistingEndpoints( - ctx context.Context, - owner ctrl_client.Object, - serviceProducer commonapi.ServiceProducer, - endpoints []*apiv1.Endpoint, - ecc EndpointCreationContext, - log logr.Logger, - ) ([]*apiv1.Endpoint, []*apiv1.Endpoint, error) -} - -func SetupEndpointIndexWithManager(mgr ctrl.Manager) error { - return mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Endpoint{}, commonapi.WorkloadOwnerKey, func(rawObj ctrl_client.Object) []string { - endpoint := rawObj.(*apiv1.Endpoint) - return slices.Map[string](endpoint.OwnerReferences, func(ref metav1.OwnerReference) string { - return string(ref.UID) - }) - }) -} - -// Attempts to create Endpoint objects for all Services exposed by the given workload object (owner parameter). -// If reservedServicePorts is not nil, it is used to determine the port to use for each Service. -// The ecc parameter contains additional context data needed for Endpoint creation, -// which is passed to the EndpointOwner implementation when Endpoints are created. -// The function is generally goroutine-safe, i.e. it can be called concurrently for different workload objects, -// but it should NOT be called concurrently for the same workload object. -func ensureEndpointsForWorkload[EndpointCreationContext any]( - ctx context.Context, - r EndpointOwner[EndpointCreationContext], - owner commonapi.DcpModelObject, - reservedServicePorts map[types.NamespacedName]int32, - ecc EndpointCreationContext, - log logr.Logger, -) { - serviceProducers, err := commonapi.GetServiceProducersForObject(owner, log) - if err != nil || len(serviceProducers) == 0 { - return - } - - var el apiv1.EndpointList - if err = r.List(ctx, &el, ctrl_client.InNamespace(owner.GetNamespace()), ctrl_client.MatchingFields{commonapi.WorkloadOwnerKey: string(owner.GetUID())}); err != nil { - log.Error(err, "Failed to list child Endpoint objects", "Workload", owner.NamespacedName().String()) - return - } - childEndpoints := toEndpointPointers(el.Items) - - for _, serviceProducer := range serviceProducers { - // Check if we have already created an Endpoint for this workload. - sweKey := ServiceWorkloadEndpointKey{ - NamespacedNameWithKind: commonapi.GetNamespacedNameWithKind(owner), - ServiceName: serviceProducer.ServiceName, - } - existingEndpoints := slices.Select(childEndpoints, func(e *apiv1.Endpoint) bool { - return e.Spec.ServiceName == serviceProducer.ServiceName && e.Spec.ServiceNamespace == serviceProducer.ServiceNamespace - }) - var cachedEndpoints []*apiv1.Endpoint - usingCache := false - if cached, found := workloadEndpointCache.Load(sweKey); found { - cachedEndpoints = toEndpointPointers(cached) - - // Our cache (if filled) takes precedence because it is most up-to-date. - diff := slices.DiffFunc(cachedEndpoints, existingEndpoints, func(a, b *apiv1.Endpoint) bool { - return a.Spec == b.Spec - }) - diff2 := slices.DiffFunc(existingEndpoints, cachedEndpoints, func(a, b *apiv1.Endpoint) bool { - return a.Spec == b.Spec - }) - usingCache = len(diff) > 0 || len(diff2) > 0 - if usingCache { - existingEndpoints = cachedEndpoints - } else { - // Kubernetes client cache is up to date, we can release ours - workloadEndpointCache.Delete(sweKey) - } - } - invalidEndpoints := []*apiv1.Endpoint{} - - svcLog := log.WithValues( - "ServiceName", serviceProducer.ServiceName, - "Workload", owner.NamespacedName().String(), - ) - - if len(existingEndpoints) > 0 { - var verr error - existingEndpoints, invalidEndpoints, verr = r.validateExistingEndpoints(ctx, owner, serviceProducer, existingEndpoints, ecc, log) - if verr != nil { - svcLog.Error(verr, "Could not validate existing Endpoint object(s)") - continue - } - } - - if len(invalidEndpoints) > 0 { - for _, endpoint := range invalidEndpoints { - deleteErr := r.Delete(ctx, endpoint, ctrl_client.PropagationPolicy(metav1.DeletePropagationBackground)) - if deleteErr != nil && !apierrors.IsNotFound(deleteErr) { - svcLog.Error(deleteErr, "Could not delete Endpoint object", - "Endpoint", endpoint.NamespacedName().String(), - ) - existingEndpoints = append(existingEndpoints, endpoint) - } - } - } - - if reservedServicePorts != nil { - if port, portFound := reservedServicePorts[serviceProducer.ServiceNamespacedName()]; portFound { - serviceProducer.Port = port - } - } - var newEndpoints []*apiv1.Endpoint - newEndpoints, err = r.createEndpoints(ctx, owner, serviceProducer, existingEndpoints, ecc, svcLog) - - if err != nil { - svcLog.Error(err, "Could not create Endpoint object(s)") - continue - } - - if len(newEndpoints) == 0 { - svcLog.V(1).Info("Endpoint(s) already exist for this workload and Service combination") - continue - } - - var createdEndpoints []*apiv1.Endpoint - for _, endpoint := range newEndpoints { - if err = ctrl.SetControllerReference(owner, endpoint, r.Scheme()); err != nil { - svcLog.Error(err, "Failed to set owner for Endpoint") - continue // Try to persist other endpoints - } - - if err = r.Create(ctx, endpoint); err != nil { - // Do not log an error if resource cleanup has started. - if !apiv1.ResourceCreationProhibited.Load() { - svcLog.Error(err, "Could not persist Endpoint object") - } - continue - } - - createdEndpoints = append(createdEndpoints, endpoint) - } - - if len(createdEndpoints) > 0 { - svcLog.V(1).Info("New Endpoint(s) created", - "Endpoints", logger.FriendlyStringSlice(slices.Map[string](createdEndpoints, (*apiv1.Endpoint).String)), - ) - } - - if usingCache || len(createdEndpoints) > 0 { - workloadEndpointCache.Store(sweKey, toEndpointValues(append(existingEndpoints, createdEndpoints...))) - } - } -} - -// Removes all Endpoint objects associated with the given workload object (owner parameter). -func removeEndpointsForWorkload(ctx context.Context, r ctrl_client.Client, owner commonapi.DcpModelObject, log logr.Logger) { - var childEndpoints apiv1.EndpointList - if err := r.List(ctx, &childEndpoints, ctrl_client.InNamespace(owner.GetNamespace()), ctrl_client.MatchingFields{commonapi.WorkloadOwnerKey: string(owner.GetUID())}); err != nil { - log.Error(err, "Failed to list child Endpoint objects", "Workload", owner.NamespacedName().String()) - } - - for _, endpoint := range childEndpoints.Items { - err := r.Delete(ctx, &endpoint, ctrl_client.PropagationPolicy(metav1.DeletePropagationBackground)) - if err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Could not delete Endpoint object", - "Endpoint", endpoint.NamespacedName().String(), - "Workload", owner.NamespacedName().String(), - ) - } - - sweKey := ServiceWorkloadEndpointKey{ - NamespacedNameWithKind: commonapi.GetNamespacedNameWithKind(owner), - ServiceName: endpoint.Spec.ServiceName, - } - - workloadEndpointCache.Delete(sweKey) - } -} - -func toEndpointPointers(evs []apiv1.Endpoint) []*apiv1.Endpoint { - return slices.Map[*apiv1.Endpoint](evs, func(e apiv1.Endpoint) *apiv1.Endpoint { return &e }) -} - -func toEndpointValues(eps []*apiv1.Endpoint) []apiv1.Endpoint { - return slices.Map[apiv1.Endpoint](eps, func(e *apiv1.Endpoint) apiv1.Endpoint { return *e }) -} +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + ctrl_client "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/pkg/commonapi" + "github.com/microsoft/dcp/pkg/logger" + "github.com/microsoft/dcp/pkg/slices" + "github.com/microsoft/dcp/pkg/syncmap" +) + +// The cache of endpoints created for a given workload and service combination. +var workloadEndpointCache *syncmap.Map[ServiceWorkloadEndpointKey, []apiv1.Endpoint] = &syncmap.Map[ServiceWorkloadEndpointKey, []apiv1.Endpoint]{} + +type ServiceWorkloadEndpointKey struct { + commonapi.NamespacedNameWithKind + ServiceName string +} + +type EndpointOwner[EndpointCreationContext any] interface { + ctrl_client.Client + + // Creates (but does not persist) new Endpoint objects for the given Service + // exposed by the workload object (owner parameter). The Service is represented by the serviceProducer parameter. + // Additional context data needed for Endpoint creation is passed in the ecc parameter. + // Existing Endpoints for the same workload and service combination are passed via the existingEndpoints parameter. + // The function should ONLY create new Endpoints that are "missing", i.e. the existing Endpoint is not sufficient. + createEndpoints( + ctx context.Context, + owner ctrl_client.Object, + serviceProducer commonapi.ServiceProducer, + existingEndpoints []*apiv1.Endpoint, + ecc EndpointCreationContext, + log logr.Logger, + ) ([]*apiv1.Endpoint, error) + + // Validates that the existing Endpoint objects correctly represent the Service exposed by the workload object. + // Returns two lists: the first list contains valid existing Endpoints that can be kept as-is, + // the second list contains invalid existing Endpoints that should be deleted. + validateExistingEndpoints( + ctx context.Context, + owner ctrl_client.Object, + serviceProducer commonapi.ServiceProducer, + endpoints []*apiv1.Endpoint, + ecc EndpointCreationContext, + log logr.Logger, + ) ([]*apiv1.Endpoint, []*apiv1.Endpoint, error) +} + +func SetupEndpointIndexWithManager(mgr ctrl.Manager) error { + return mgr.GetFieldIndexer().IndexField(context.Background(), &apiv1.Endpoint{}, commonapi.WorkloadOwnerKey, func(rawObj ctrl_client.Object) []string { + endpoint := rawObj.(*apiv1.Endpoint) + return slices.Map[string](endpoint.OwnerReferences, func(ref metav1.OwnerReference) string { + return string(ref.UID) + }) + }) +} + +// Attempts to create Endpoint objects for all Services exposed by the given workload object (owner parameter). +// If reservedServicePorts is not nil, it is used to determine the port to use for each Service. +// The ecc parameter contains additional context data needed for Endpoint creation, +// which is passed to the EndpointOwner implementation when Endpoints are created. +// The function is generally goroutine-safe, i.e. it can be called concurrently for different workload objects, +// but it should NOT be called concurrently for the same workload object. +func ensureEndpointsForWorkload[EndpointCreationContext any]( + ctx context.Context, + r EndpointOwner[EndpointCreationContext], + owner commonapi.DcpModelObject, + reservedServicePorts map[types.NamespacedName]int32, + ecc EndpointCreationContext, + log logr.Logger, +) { + serviceProducers, err := commonapi.GetServiceProducersForObject(owner, log) + if err != nil || len(serviceProducers) == 0 { + return + } + + var el apiv1.EndpointList + if err = r.List(ctx, &el, ctrl_client.InNamespace(owner.GetNamespace()), ctrl_client.MatchingFields{commonapi.WorkloadOwnerKey: string(owner.GetUID())}); err != nil { + log.Error(err, "Failed to list child Endpoint objects", "Workload", owner.NamespacedName().String()) + return + } + childEndpoints := toEndpointPointers(el.Items) + + for _, serviceProducer := range serviceProducers { + // Check if we have already created an Endpoint for this workload. + sweKey := ServiceWorkloadEndpointKey{ + NamespacedNameWithKind: commonapi.GetNamespacedNameWithKind(owner), + ServiceName: serviceProducer.ServiceName, + } + existingEndpoints := slices.Select(childEndpoints, func(e *apiv1.Endpoint) bool { + return e.Spec.ServiceName == serviceProducer.ServiceName && e.Spec.ServiceNamespace == serviceProducer.ServiceNamespace + }) + var cachedEndpoints []*apiv1.Endpoint + usingCache := false + if cached, found := workloadEndpointCache.Load(sweKey); found { + cachedEndpoints = toEndpointPointers(cached) + + // Our cache (if filled) takes precedence because it is most up-to-date. + diff := slices.DiffFunc(cachedEndpoints, existingEndpoints, func(a, b *apiv1.Endpoint) bool { + return a.Spec == b.Spec + }) + diff2 := slices.DiffFunc(existingEndpoints, cachedEndpoints, func(a, b *apiv1.Endpoint) bool { + return a.Spec == b.Spec + }) + usingCache = len(diff) > 0 || len(diff2) > 0 + if usingCache { + existingEndpoints = cachedEndpoints + } else { + // Kubernetes client cache is up to date, we can release ours + workloadEndpointCache.Delete(sweKey) + } + } + invalidEndpoints := []*apiv1.Endpoint{} + + svcLog := log.WithValues( + "ServiceName", serviceProducer.ServiceName, + "Workload", owner.NamespacedName().String(), + ) + + if len(existingEndpoints) > 0 { + var verr error + existingEndpoints, invalidEndpoints, verr = r.validateExistingEndpoints(ctx, owner, serviceProducer, existingEndpoints, ecc, log) + if verr != nil { + svcLog.Error(verr, "Could not validate existing Endpoint object(s)") + continue + } + } + + if len(invalidEndpoints) > 0 { + for _, endpoint := range invalidEndpoints { + deleteErr := r.Delete(ctx, endpoint, ctrl_client.PropagationPolicy(metav1.DeletePropagationBackground)) + if deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + svcLog.Error(deleteErr, "Could not delete Endpoint object", + "Endpoint", endpoint.NamespacedName().String(), + ) + existingEndpoints = append(existingEndpoints, endpoint) + } + } + } + + if reservedServicePorts != nil { + if port, portFound := reservedServicePorts[serviceProducer.ServiceNamespacedName()]; portFound { + serviceProducer.Port = port + } + } + var newEndpoints []*apiv1.Endpoint + newEndpoints, err = r.createEndpoints(ctx, owner, serviceProducer, existingEndpoints, ecc, svcLog) + + if err != nil { + svcLog.Error(err, "Could not create Endpoint object(s)") + continue + } + + if len(newEndpoints) == 0 { + svcLog.V(1).Info("Endpoint(s) already exist for this workload and Service combination") + continue + } + + var createdEndpoints []*apiv1.Endpoint + for _, endpoint := range newEndpoints { + if err = ctrl.SetControllerReference(owner, endpoint, r.Scheme()); err != nil { + svcLog.Error(err, "Failed to set owner for Endpoint") + continue // Try to persist other endpoints + } + + if err = r.Create(ctx, endpoint); err != nil { + // Do not log an error if resource cleanup has started. + if !apiv1.ResourceCreationProhibited.Load() { + svcLog.Error(err, "Could not persist Endpoint object") + } + continue + } + + createdEndpoints = append(createdEndpoints, endpoint) + } + + if len(createdEndpoints) > 0 { + svcLog.V(1).Info("New Endpoint(s) created", + "Endpoints", logger.FriendlyStringSlice(slices.Map[string](createdEndpoints, (*apiv1.Endpoint).String)), + ) + } + + if usingCache || len(createdEndpoints) > 0 { + workloadEndpointCache.Store(sweKey, toEndpointValues(append(existingEndpoints, createdEndpoints...))) + } + } +} + +// Removes all Endpoint objects associated with the given workload object (owner parameter). +func removeEndpointsForWorkload(ctx context.Context, r ctrl_client.Client, owner commonapi.DcpModelObject, log logr.Logger) { + var childEndpoints apiv1.EndpointList + if err := r.List(ctx, &childEndpoints, ctrl_client.InNamespace(owner.GetNamespace()), ctrl_client.MatchingFields{commonapi.WorkloadOwnerKey: string(owner.GetUID())}); err != nil { + log.Error(err, "Failed to list child Endpoint objects", "Workload", owner.NamespacedName().String()) + } + + for _, endpoint := range childEndpoints.Items { + err := r.Delete(ctx, &endpoint, ctrl_client.PropagationPolicy(metav1.DeletePropagationBackground)) + if err != nil && !apierrors.IsNotFound(err) { + log.Error(err, "Could not delete Endpoint object", + "Endpoint", endpoint.NamespacedName().String(), + "Workload", owner.NamespacedName().String(), + ) + } + + sweKey := ServiceWorkloadEndpointKey{ + NamespacedNameWithKind: commonapi.GetNamespacedNameWithKind(owner), + ServiceName: endpoint.Spec.ServiceName, + } + + workloadEndpointCache.Delete(sweKey) + } +} + +func toEndpointPointers(evs []apiv1.Endpoint) []*apiv1.Endpoint { + return slices.Map[*apiv1.Endpoint](evs, func(e apiv1.Endpoint) *apiv1.Endpoint { return &e }) +} + +func toEndpointValues(eps []*apiv1.Endpoint) []apiv1.Endpoint { + return slices.Map[apiv1.Endpoint](eps, func(e *apiv1.Endpoint) apiv1.Endpoint { return *e }) +} diff --git a/controllers/executable_controller.go b/controllers/executable_controller.go index 75662e06..2b988998 100644 --- a/controllers/executable_controller.go +++ b/controllers/executable_controller.go @@ -30,6 +30,7 @@ import ( "github.com/microsoft/dcp/internal/health" "github.com/microsoft/dcp/internal/logs" "github.com/microsoft/dcp/internal/networking" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/internal/templating" "github.com/microsoft/dcp/pkg/commonapi" "github.com/microsoft/dcp/pkg/concurrency" @@ -63,6 +64,11 @@ var executableStateInitializers = map[apiv1.ExecutableState]executableStateIniti type runStateMap = ObjectStateMap[RunID, ExecutableRunInfo, *ExecutableRunInfo, *apiv1.Executable] +type ExecutableReconcilerConfig struct { + StateStore *statestore.Store + ResourceLeaseOwner process.ProcessTreeItem +} + // ExecutableReconciler reconciles a Executable object type ExecutableReconciler struct { *ReconcilerBase[apiv1.Executable, *apiv1.Executable] @@ -81,6 +87,8 @@ type ExecutableReconciler struct { // A WorkQueue for operations related to stopping Executables (which might take a while). stopQueue *resiliency.WorkQueue + + config ExecutableReconcilerConfig } var ( @@ -102,6 +110,26 @@ func NewExecutableReconciler( log logr.Logger, executableRunners map[apiv1.ExecutionType]ExecutableRunner, healthProbeSet *health.HealthProbeSet, +) *ExecutableReconciler { + return NewExecutableReconcilerWithConfig( + lifetimeCtx, + client, + noCacheClient, + log, + executableRunners, + healthProbeSet, + ExecutableReconcilerConfig{}, + ) +} + +func NewExecutableReconcilerWithConfig( + lifetimeCtx context.Context, + client ctrl_client.Client, + noCacheClient ctrl_client.Reader, + log logr.Logger, + executableRunners map[apiv1.ExecutionType]ExecutableRunner, + healthProbeSet *health.HealthProbeSet, + config ExecutableReconcilerConfig, ) *ExecutableReconciler { base := NewReconcilerBase[apiv1.Executable](client, noCacheClient, log, lifetimeCtx) @@ -112,6 +140,7 @@ func NewExecutableReconciler( hpSet: healthProbeSet, healthProbeCh: concurrency.NewUnboundedChan[health.HealthProbeReport](lifetimeCtx), stopQueue: resiliency.NewWorkQueue(lifetimeCtx, maxParallelExecutableStops), + config: config, } go r.handleHealthProbeResults() @@ -195,6 +224,18 @@ func (r *ExecutableReconciler) Reconcile(ctx context.Context, req ctrl.Request) func (r *ExecutableReconciler) handleDeletionRequest(ctx context.Context, exe *apiv1.Executable, log logr.Logger) objectChange { _, runInfo := r.runs.BorrowByNamespacedName(exe.NamespacedName()) + if exe.Spec.Persistent { + log.V(1).Info("Persistent Executable is being deleted; leaving underlying process running") + var deletionChange objectChange + leaseChange := r.withPersistentExecutableLease(ctx, exe, log, func(context.Context, *statestore.ResourceLease) objectChange { + r.releasePersistentExecutableResources(ctx, exe, runInfo, log) + r.runs.DeleteByNamespacedName(exe.NamespacedName()) + deletionChange = deleteFinalizer(exe, executableFinalizer, log) + return deletionChange + }) + return leaseChange | deletionChange + } + var change objectChange switch { @@ -237,8 +278,12 @@ func (r *ExecutableReconciler) manageExecutable(ctx context.Context, exe *apiv1. // Even if Executable.State == targetExecutableState, we still want to run the initializer // to ensure that Executable.Status is up to date and that the real-world resources // associated with Executable object are up to date. - initializer := getStateInitializer(executableStateInitializers, targetExecutableState, log) - change := initializer(ctx, r, exe, targetExecutableState, runInfo, log) + runInitializer := func(ctx context.Context) objectChange { + initializer := getStateInitializer(executableStateInitializers, targetExecutableState, log) + return initializer(ctx, r, exe, targetExecutableState, runInfo, log) + } + + change := runInitializer(ctx) if runInfo != nil { r.runs.Update(exe.NamespacedName(), runInfo.RunID, runInfo) @@ -261,7 +306,39 @@ func handleNewExecutable( return statusChanged } - return r.startExecutable(ctx, exe, runInfo, log) + if exe.Spec.Persistent { + leaseErr := r.acquirePersistentExecutableResourceLease(ctx, exe, log) + if errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + return additionalReconciliationNeeded + } + if leaseErr != nil { + log.Error(leaseErr, "Could not acquire persistent Executable resource lease") + return r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + } + + if !exe.ShouldStart() && exe.Status.State == apiv1.ExecutableStateEmpty && runInfo == nil { + if exe.Spec.Persistent { + adopted, adoptionChange := r.tryAdoptExistingPersistentExecutable(ctx, exe, log) + _ = r.releasePersistentExecutableResourceLease(ctx, exe, log, false) + if adopted || adoptionChange != noChange { + return adoptionChange + } + } + + // We should wait to create an executable until the user clears Start = false + log.V(1).Info("Waiting for the executable to be started") + return noChange + } + + change := r.startExecutable(ctx, exe, runInfo, log) + if exe.Spec.Persistent { + _, updatedRunInfo := r.runs.BorrowByNamespacedName(exe.NamespacedName()) + if !persistentExecutableStartupInProgress(updatedRunInfo) { + _ = r.releasePersistentExecutableResourceLease(ctx, exe, log, false) + } + } + return change } func ensureExecutableRunningState( @@ -280,6 +357,23 @@ func ensureExecutableRunningState( return change } + if exe.Spec.Persistent && runInfo.Pid != apiv1.UnknownPID && !runInfo.ProcessIdentityTime.IsZero() { + pid, pidErr := process.Int64_ToPidT(*runInfo.Pid) + if pidErr != nil { + log.Error(pidErr, "Persistent Executable run has invalid PID") + runInfo.ExeState = apiv1.ExecutableStateUnknown + return change | r.setExecutableState(exe, apiv1.ExecutableStateUnknown) + } + if _, findErr := process.FindProcess(pid, runInfo.ProcessIdentityTime); findErr != nil { + log.Info("Persistent Executable process is no longer running", "PID", pid, "Error", findErr.Error()) + r.deletePersistentProcessRecord(ctx, exe.NamespacedName(), log) + runInfo.ExeState = apiv1.ExecutableStateFinished + runInfo.FinishTimestamp = metav1.NowMicro() + r.disableEndpointsAndHealthProbes(ctx, exe, runInfo, log) + return change | runInfo.ApplyTo(exe, log) | r.setExecutableState(exe, apiv1.ExecutableStateFinished) + } + } + if exe.Spec.Stop { runInfo.ExeState = apiv1.ExecutableStateStopping change |= r.setExecutableState(exe, apiv1.ExecutableStateStopping) @@ -352,6 +446,14 @@ func (r *ExecutableReconciler) startExecutable( ri *ExecutableRunInfo, log logr.Logger, ) objectChange { + if leaseErr := r.verifyPersistentExecutableResourceLeaseHeld(ctx, exe, log); leaseErr != nil { + if ri != nil { + ri.ExeState = apiv1.ExecutableStateFailedToStart + r.runs.Update(exe.NamespacedName(), ri.RunID, ri.Clone()) + } + return r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + if ri == nil { log.V(1).Info("Starting Executable...") ri = NewRunInfo(exe) @@ -365,6 +467,8 @@ func (r *ExecutableReconciler) startExecutable( prepareCertErr := r.prepareCertificateFiles(exe, log) if prepareCertErr != nil { // Error already logged in prepareCertificateFiles() + ri.ExeState = apiv1.ExecutableStateFailedToStart + r.runs.Update(exe.NamespacedName(), ri.RunID, ri.Clone()) r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) return statusChanged } @@ -383,6 +487,13 @@ func (r *ExecutableReconciler) startExecutable( } } + if exe.Spec.Persistent && len(ri.startResults) == 0 { + adopted, adoptionChange := r.tryAdoptPersistentExecutable(ctx, exe, ri, log) + if adopted || adoptionChange != noChange { + return adoptionChange | environmentChanged + } + } + if ri.startupStage >= StartupStageDefaultRunner && len(ri.startResults) == 0 { // Should never happen log.Error(errors.New("inconsistent Executable startup state"), "No startup results found despite startup stage indicating that a startup attempt was made") r.setExecutableState(exe, apiv1.ExecutableStateUnknown) @@ -401,7 +512,27 @@ func (r *ExecutableReconciler) startExecutable( } // Helper function to update Executable status when the start attempt was successful. - handleSuccessfulStart := func(res *ExecutableStartResult) { + handleSuccessfulStart := func(res *ExecutableStartResult) objectChange { + if exe.Spec.Persistent { + persistErr := r.upsertPersistentProcessRecord(ctx, exe, res) + if persistErr != nil { + log.Error(persistErr, "Could not persist Executable process record") + r.cleanUpPersistentStartAfterRecordFailure(ctx, exe, ri.startupStage, res, log) + + now := metav1.NowMicro() + ri.ExeState = apiv1.ExecutableStateFailedToStart + ri.Pid = apiv1.UnknownPID + ri.RunID = UnknownRunID + ri.StartupTimestamp = now + ri.FinishTimestamp = now + change := ri.ApplyTo(exe, log) + exe.Status.ExecutionID = "" + exe.Status.PID = apiv1.UnknownPID + r.runs.DeleteByNamespacedName(exe.NamespacedName()) + return change | r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + } + startingRunID := ri.RunID // Might be the temporary starting run ID. We need that to update the runs map. res.applyTo(ri) ri.ApplyTo(exe, log) @@ -411,6 +542,7 @@ func (r *ExecutableReconciler) startExecutable( if res.ExeState == apiv1.ExecutableStateRunning && res.StartWaitForRunCompletion != nil { res.StartWaitForRunCompletion() } + return statusChanged } var unknownStartupFailureErr error = errors.New("the reason for the Executable startup failure is unknown") @@ -419,8 +551,7 @@ func (r *ExecutableReconciler) startExecutable( // or because the previous, asynchronous attempt has completed (successfully or not). // So first we need to check the result of the previous startup attempt (if any). if startResult.IsSuccessfullyCompleted() { - handleSuccessfulStart(startResult) - return statusChanged | environmentChanged + return handleSuccessfulStart(startResult) | environmentChanged } else if startResult != nil { runErr := startResult.StartupError if runErr != nil { @@ -475,8 +606,7 @@ func (r *ExecutableReconciler) startExecutable( } if startResult.IsSuccessfullyCompleted() { - handleSuccessfulStart(startResult) - return statusChanged | environmentChanged + return handleSuccessfulStart(startResult) | environmentChanged } // Asynchronous start in progress, wait for OnStartupCompleted() to be called @@ -484,6 +614,39 @@ func (r *ExecutableReconciler) startExecutable( } } +func (r *ExecutableReconciler) cleanUpPersistentStartAfterRecordFailure( + ctx context.Context, + exe *apiv1.Executable, + startupStage ExecutableStartuptStage, + res *ExecutableStartResult, + log logr.Logger, +) { + if res == nil { + return + } + + if res.RunID != UnknownRunID { + runner, runnerNotFoundErr := r.getExecutableRunner(exe, startupStage) + if runnerNotFoundErr != nil { + log.Error(runnerNotFoundErr, "The persistent Executable cannot be stopped after process record update failed") + } else if stopErr := runner.StopRun(ctx, res.RunID, log); stopErr != nil { + log.Error(stopErr, "Could not stop persistent Executable after process record update failed", "RunID", res.RunID) + } + } + + removeOutputFile := func(path string, stream string) { + if path == "" || osutil.EnvVarSwitchEnabled(usvc_io.DCP_PRESERVE_EXECUTABLE_LOGS) { + return + } + if removeErr := logs.RemoveWithRetry(ctx, path); removeErr != nil { + log.Error(removeErr, "Could not remove persistent Executable output file after process record update failed", "Path", path, "Stream", stream) + } + } + + removeOutputFile(res.StdOutFile, "stdout") + removeOutputFile(res.StdErrFile, "stderr") +} + func (r *ExecutableReconciler) OnMainProcessChanged(runID RunID, pid process.Pid_t) { r.processRunChangeNotification("Run process changed", runID, pid, apiv1.UnknownExitCode, nil, func(ri *ExecutableRunInfo) { ri.ExeState = apiv1.ExecutableStateRunning @@ -548,9 +711,13 @@ func (r *ExecutableReconciler) processRunChangeNotification( } runMap := r.runs - r.runs.QueueDeferredOp(name, func(types.NamespacedName, RunID, *apiv1.Executable) { + runIsTerminal := runInfo.ExeState.IsTerminal() + r.runs.QueueDeferredOp(name, func(_ types.NamespacedName, _ RunID, exe *apiv1.Executable) { // The run may have been deleted by the time we get here, so we do not care if Update() returns false. _ = runMap.Update(name, runID, runInfo) + if runIsTerminal && exe != nil && exe.Spec.Persistent { + r.deletePersistentProcessRecordWithLease(context.Background(), name, log) + } }) r.ScheduleReconciliation(name) @@ -1196,11 +1363,29 @@ func (r *ExecutableReconciler) setExecutableState(exe *apiv1.Executable, state a } } + if exe.Spec.Persistent && persistentExecutableLeaseReleaseState(state) { + // Intentionally ignore errors: this is a defensive, idempotent release attempt for stable states. + _ = r.releasePersistentExecutableResourceLease(context.Background(), exe, r.Log, true) + } + change |= updateExecutableHealthStatus(exe, state, r.Log) return change } +func persistentExecutableLeaseReleaseState(state apiv1.ExecutableState) bool { + switch state { + case apiv1.ExecutableStateRunning, + apiv1.ExecutableStateTerminated, + apiv1.ExecutableStateFailedToStart, + apiv1.ExecutableStateFinished, + apiv1.ExecutableStateUnknown: + return true + default: + return false + } +} + func updateExecutableHealthStatus(exe *apiv1.Executable, state apiv1.ExecutableState, log logr.Logger) objectChange { var newHealthStatus apiv1.HealthStatus diff --git a/controllers/executable_persistence.go b/controllers/executable_persistence.go new file mode 100644 index 00000000..42eee648 --- /dev/null +++ b/controllers/executable_persistence.go @@ -0,0 +1,592 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package controllers + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "reflect" + stdslices "slices" + "strings" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/statestore" + "github.com/microsoft/dcp/pkg/logger" + "github.com/microsoft/dcp/pkg/maps" + "github.com/microsoft/dcp/pkg/pointers" + "github.com/microsoft/dcp/pkg/process" +) + +func (r *ExecutableReconciler) withPersistentExecutableLease(ctx context.Context, exe *apiv1.Executable, log logr.Logger, f func(context.Context, *statestore.ResourceLease) objectChange) objectChange { + if f == nil { + log.Error(fmt.Errorf("persistent Executable lease callback cannot be nil"), "Could not acquire persistent Executable resource lease") + return r.setExecutableState(exe, apiv1.ExecutableStateUnknown) + } + + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + log.Error(stateStoreErr, "Persistent Executable cannot be reconciled without a state store") + return r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + leaseOwner, leaseOwnerErr := r.getResourceLeaseOwner() + if leaseOwnerErr != nil { + log.Error(leaseOwnerErr, "Could not determine persistent Executable resource lease owner") + return r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + var change objectChange + leaseErr := stateStore.WithResourceLease(ctx, exe, leaseOwner, resourceLeaseRevalidationInterval, func(ctx context.Context, lease *statestore.ResourceLease) error { + log.V(1).Info("Acquired resource lease", "ResourceKey", lease.ResourceKey) + change = f(ctx, lease) + return nil + }) + if errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + logResourceLeaseHeld(log, leaseErr, exe.GetLeaseKey(), "Persistent Executable is being updated by another DCP instance, retrying") + return additionalReconciliationNeeded + } + if leaseErr != nil { + log.Error(leaseErr, "Could not acquire persistent Executable resource lease", "ResourceKey", exe.GetLeaseKey()) + return r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + return change +} + +func (r *ExecutableReconciler) acquirePersistentExecutableResourceLease(ctx context.Context, exe *apiv1.Executable, log logr.Logger) error { + if !exe.Spec.Persistent { + return nil + } + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + return stateStoreErr + } + leaseOwner, leaseOwnerErr := r.getResourceLeaseOwner() + if leaseOwnerErr != nil { + return leaseOwnerErr + } + + lease, leaseErr := stateStore.AcquireResourceLease(ctx, exe, leaseOwner, resourceLeaseRevalidationInterval) + if errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + logResourceLeaseHeld(log, leaseErr, exe.GetLeaseKey(), "Persistent Executable is being updated by another DCP instance, retrying") + } + if leaseErr != nil { + return leaseErr + } + + log.V(1).Info("Acquired resource lease", "ResourceKey", lease.ResourceKey) + return nil +} + +func (r *ExecutableReconciler) verifyPersistentExecutableResourceLeaseHeld(ctx context.Context, exe *apiv1.Executable, log logr.Logger) error { + if !exe.Spec.Persistent { + return nil + } + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + return stateStoreErr + } + leaseOwner, leaseOwnerErr := r.getResourceLeaseOwner() + if leaseOwnerErr != nil { + return leaseOwnerErr + } + + leaseErr := stateStore.VerifyResourceLeaseHeld(ctx, exe, leaseOwner) + if leaseErr != nil { + log.Error(leaseErr, "Cannot continue persistent Executable startup because this DCP instance does not hold the resource lease") + return leaseErr + } + + return nil +} + +func (r *ExecutableReconciler) releasePersistentExecutableResourceLease( + ctx context.Context, + exe *apiv1.Executable, + log logr.Logger, + suppressNotHeldLog bool, +) error { + if !exe.Spec.Persistent { + return nil + } + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + return stateStoreErr + } + leaseOwner, leaseOwnerErr := r.getResourceLeaseOwner() + if leaseOwnerErr != nil { + return leaseOwnerErr + } + + releaseErr := stateStore.ReleaseResourceLease(ctx, exe, leaseOwner) + if releaseErr != nil { + if errors.Is(releaseErr, statestore.ErrResourceLeaseNotHeld) { + logResourceLeaseNotHeld(log, suppressNotHeldLog, exe.GetLeaseKey(), "Persistent Executable resource lease was not held") + return releaseErr + } + log.Error(releaseErr, "Could not release persistent Executable resource lease") + return releaseErr + } + + return nil +} + +func persistentExecutableStartupInProgress(runInfo *ExecutableRunInfo) bool { + return runInfo != nil && runInfo.ExeState == apiv1.ExecutableStateStarting +} + +func (r *ExecutableReconciler) tryAdoptExistingPersistentExecutable(ctx context.Context, exe *apiv1.Executable, log logr.Logger) (bool, objectChange) { + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + log.Error(stateStoreErr, "Persistent Executable cannot be reconciled without a state store") + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + record, recordErr := stateStore.GetPersistentProcess(ctx, exe.GetLeaseKey()) + if errors.Is(recordErr, statestore.ErrPersistentProcessNotFound) { + return false, noChange + } + if recordErr != nil { + log.Error(recordErr, "Could not read persistent Executable process record", "ResourceKey", exe.GetLeaseKey()) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + ri := NewRunInfo(exe) + computed, environmentChange := r.computeExecutableEnvironment(ctx, exe, log, ri) + if !computed { + return false, environmentChange + } + + adopted, adoptionChange := r.tryAdoptPersistentExecutableRecord(ctx, exe, ri, record, log) + return adopted, environmentChange | adoptionChange +} + +func (r *ExecutableReconciler) tryAdoptPersistentExecutable(ctx context.Context, exe *apiv1.Executable, runInfo *ExecutableRunInfo, log logr.Logger) (bool, objectChange) { + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + log.Error(stateStoreErr, "Persistent Executable cannot be reconciled without a state store") + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + resourceKey := exe.GetLeaseKey() + record, recordErr := stateStore.GetPersistentProcess(ctx, resourceKey) + if errors.Is(recordErr, statestore.ErrPersistentProcessNotFound) { + return false, noChange + } + if recordErr != nil { + log.Error(recordErr, "Could not read persistent Executable process record", "ResourceKey", resourceKey) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + return r.tryAdoptPersistentExecutableRecord(ctx, exe, runInfo, record, log) +} + +func (r *ExecutableReconciler) tryAdoptPersistentExecutableRecord(ctx context.Context, exe *apiv1.Executable, runInfo *ExecutableRunInfo, record *statestore.PersistentProcessRecord, log logr.Logger) (bool, objectChange) { + resourceKey := exe.GetLeaseKey() + lifecycleInfo, lifecycleKeyErr := persistentExecutableLifecycleInfo(exe) + if lifecycleKeyErr != nil { + log.Error(lifecycleKeyErr, "Could not calculate Executable lifecycle key") + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + if record.LifecycleKey != lifecycleInfo.Key { + if lifecycleInfo.HasDefaultKey { + args, env, other := calculatePersistentExecutableChanges(record.LifecycleMetadata, lifecycleInfo.Metadata) + log.Info("Found existing persistent Executable process record, but calculated lifecycle key does not match", + "OldLifecycleKey", record.LifecycleKey, + "NewLifecycleKey", lifecycleInfo.Key, + "ArgChanges", args, + "EnvChanges", env, + "OtherChanges", other) + } else { + log.Info("Found existing persistent Executable process record, but custom lifecycle key does not match", + "OldLifecycleKey", record.LifecycleKey, + "NewLifecycleKey", lifecycleInfo.Key) + } + if _, findErr := process.FindProcess(record.PID, record.IdentityTime); findErr == nil { + stopErr := r.stopPersistentExecutableRecord(ctx, exe, record, log) + if stopErr != nil { + log.Error(stopErr, "Could not stop persistent Executable process with stale lifecycle key", "PID", record.PID) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + } else { + log.Info("Persistent Executable process record with stale lifecycle key is no longer running", + "PID", record.PID, + "Error", findErr.Error()) + } + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + log.Error(stateStoreErr, "Could not open state store to delete stale persistent Executable process record", "ResourceKey", resourceKey) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + if deleteErr := stateStore.DeletePersistentProcess(ctx, resourceKey); deleteErr != nil { + log.Error(deleteErr, "Could not delete stale persistent Executable process record", "ResourceKey", resourceKey) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + return false, noChange + } + + if _, findErr := process.FindProcess(record.PID, record.IdentityTime); findErr != nil { + log.Info("Persistent Executable process record is stale; process is no longer running", + "PID", record.PID, + "Error", findErr.Error()) + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + log.Error(stateStoreErr, "Could not open state store to delete stale persistent Executable process record", "ResourceKey", resourceKey) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + if deleteErr := stateStore.DeletePersistentProcess(ctx, resourceKey); deleteErr != nil { + log.Error(deleteErr, "Could not delete stale persistent Executable process record", "ResourceKey", resourceKey) + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + return false, noChange + } + displayStartTime := process.StartTimeForProcess(record.PID) + + runner, runnerErr := r.getExecutableRunner(exe, StartupStageDefaultRunner) + if runnerErr != nil { + log.Error(runnerErr, "The persistent Executable runner is not available") + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + persistentRunner, ok := runner.(PersistentExecutableRunner) + if !ok { + log.Error(fmt.Errorf("runner does not support persistent run adoption"), "The persistent Executable cannot be adopted") + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + runID := RunID(record.RunID) + adoptionErr := persistentRunner.AdoptRun(ctx, ExecutableRunAdoptionInfo{ + RunID: runID, + Pid: record.PID, + ProcessIdentityTime: record.IdentityTime, + StdOutFile: record.StdOutFile, + StdErrFile: record.StdErrFile, + CommandInfo: exe.Spec.ExecutablePath, + }, r, log) + if adoptionErr != nil { + log.Error(adoptionErr, "Could not adopt persistent Executable process") + return false, r.setExecutableState(exe, apiv1.ExecutableStateFailedToStart) + } + + ri := runInfo + if ri == nil { + ri = NewRunInfo(exe) + } + startingRunID := ri.RunID + ri.ExeState = apiv1.ExecutableStateRunning + ri.RunID = runID + pointers.SetValue(&ri.Pid, int64(record.PID)) + ri.ProcessIdentityTime = record.IdentityTime + ri.StartupTimestamp = metav1.NewMicroTime(displayStartTime) + ri.StdOutFile = record.StdOutFile + ri.StdErrFile = record.StdErrFile + ri.startupStage = StartupStageDefaultRunner + if startingRunID == UnknownRunID { + r.runs.Store(exe.NamespacedName(), ri.RunID, ri.Clone()) + } else { + r.runs.UpdateChangingStateKey(exe.NamespacedName(), startingRunID, ri.RunID, ri.Clone()) + } + + log.Info("Adopted persistent Executable process", "PID", record.PID, "RunID", runID) + return true, ri.ApplyTo(exe, log) | r.setExecutableState(exe, apiv1.ExecutableStateRunning) +} + +func (r *ExecutableReconciler) stopPersistentExecutableRecord(ctx context.Context, exe *apiv1.Executable, record *statestore.PersistentProcessRecord, log logr.Logger) error { + runner, runnerErr := r.getExecutableRunner(exe, StartupStageDefaultRunner) + if runnerErr != nil { + return fmt.Errorf("persistent Executable runner is not available: %w", runnerErr) + } + persistentRunner, ok := runner.(PersistentExecutableRunner) + if !ok { + return fmt.Errorf("runner does not support persistent run adoption") + } + + runID := RunID(record.RunID) + adoptionErr := persistentRunner.AdoptRun(ctx, ExecutableRunAdoptionInfo{ + RunID: runID, + Pid: record.PID, + ProcessIdentityTime: record.IdentityTime, + StdOutFile: record.StdOutFile, + StdErrFile: record.StdErrFile, + CommandInfo: exe.Spec.ExecutablePath, + }, nil, log) + if adoptionErr != nil { + return fmt.Errorf("could not adopt persistent Executable process before stopping it: %w", adoptionErr) + } + + return runner.StopRun(ctx, runID, log) +} + +func (r *ExecutableReconciler) upsertPersistentProcessRecord(ctx context.Context, exe *apiv1.Executable, res *ExecutableStartResult) error { + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + return stateStoreErr + } + if res.Pid == apiv1.UnknownPID { + return fmt.Errorf("cannot persist Executable process record without a valid PID") + } + + pid, pidErr := process.Int64_ToPidT(*res.Pid) + if pidErr != nil { + return fmt.Errorf("cannot persist Executable process record with invalid PID %d: %w", *res.Pid, pidErr) + } + + lifecycleInfo, lifecycleKeyErr := persistentExecutableLifecycleInfo(exe) + if lifecycleKeyErr != nil { + return fmt.Errorf("could not calculate Executable lifecycle key: %w", lifecycleKeyErr) + } + + return stateStore.UpsertPersistentProcess(ctx, statestore.PersistentProcessRecord{ + ResourceKey: exe.GetLeaseKey(), + LifecycleKey: lifecycleInfo.Key, + PID: pid, + IdentityTime: res.ProcessIdentityTime, + RunID: string(res.RunID), + StdOutFile: res.StdOutFile, + StdErrFile: res.StdErrFile, + LifecycleMetadata: lifecycleInfo.Metadata, + }) +} + +type persistentExecutableLifecycleData struct { + Key string + HasDefaultKey bool + Metadata string +} + +type persistentExecutableLifecycleMetadata struct { + ExecutablePath string `json:"executablePath"` + WorkingDirectory string `json:"workingDirectory,omitempty"` + AmbientEnvironment string `json:"ambientEnvironment,omitempty"` + EffectiveArgs []string `json:"effectiveArgs,omitempty"` + ExplicitEffectiveEnv []persistentExecutableEnvMetadata `json:"explicitEffectiveEnv,omitempty"` + PEMCertificates []string `json:"pemCertificates,omitempty"` + PEMCertificatesContinueOnError bool `json:"pemCertificatesContinueOnError,omitempty"` +} + +type persistentExecutableEnvMetadata struct { + Name string `json:"name"` + ValueHash string `json:"valueHash"` +} + +func persistentExecutableLifecycleInfo(exe *apiv1.Executable) (persistentExecutableLifecycleData, error) { + lifecycleKey, hasDefaultLifecycleKey, lifecycleKeyErr := exe.GetLifecycleKey() + if lifecycleKeyErr != nil { + return persistentExecutableLifecycleData{}, lifecycleKeyErr + } + if !hasDefaultLifecycleKey { + return persistentExecutableLifecycleData{ + Key: lifecycleKey, + HasDefaultKey: false, + }, nil + } + + lifecycleSpec, lifecycleSpecErr := exe.EffectiveLifecycleSpec() + if lifecycleSpecErr != nil { + return persistentExecutableLifecycleData{}, lifecycleSpecErr + } + lifecycleMetadata := persistentExecutableLifecycleMetadata{ + ExecutablePath: lifecycleSpec.ExecutablePath, + WorkingDirectory: lifecycleSpec.WorkingDirectory, + AmbientEnvironment: string(lifecycleSpec.AmbientEnvironment.Behavior), + EffectiveArgs: stdslices.Clone(lifecycleSpec.Args), + } + + lifecycleMetadata.ExplicitEffectiveEnv = make([]persistentExecutableEnvMetadata, 0, len(lifecycleSpec.Env)) + for _, envVar := range lifecycleSpec.Env { + lifecycleMetadata.ExplicitEffectiveEnv = append(lifecycleMetadata.ExplicitEffectiveEnv, persistentExecutableEnvMetadata{ + Name: envVar.Name, + ValueHash: fmt.Sprintf("%x", sha256.Sum256([]byte(envVar.Value))), + }) + } + + if lifecycleSpec.PemCertificates != nil { + sortedPemCertificates := stdslices.Clone(lifecycleSpec.PemCertificates.Certificates) + stdslices.SortFunc(sortedPemCertificates, func(c1, c2 apiv1.PemCertificate) int { + return strings.Compare(c1.Thumbprint, c2.Thumbprint) + }) + + for i := range sortedPemCertificates { + lifecycleMetadata.PEMCertificates = append(lifecycleMetadata.PEMCertificates, sortedPemCertificates[i].Thumbprint) + } + lifecycleMetadata.PEMCertificatesContinueOnError = lifecycleSpec.PemCertificates.ContinueOnError + } + + metadataBytes, metadataErr := json.Marshal(lifecycleMetadata) + if metadataErr != nil { + return persistentExecutableLifecycleData{}, metadataErr + } + + return persistentExecutableLifecycleData{ + Key: lifecycleKey, + HasDefaultKey: true, + Metadata: string(metadataBytes), + }, nil +} + +func calculatePersistentExecutableChanges(oldMetadata, newMetadata string) (args []string, env []string, other []string) { + if oldMetadata == "" { + return nil, nil, []string{"Executable lifecycle metadata was not recorded for the existing process"} + } + + var oldLifecycleMetadata persistentExecutableLifecycleMetadata + var newLifecycleMetadata persistentExecutableLifecycleMetadata + oldMetadataErr := json.Unmarshal([]byte(oldMetadata), &oldLifecycleMetadata) + newMetadataErr := json.Unmarshal([]byte(newMetadata), &newLifecycleMetadata) + if oldMetadataErr != nil || newMetadataErr != nil { + return nil, nil, []string{"Executable lifecycle metadata could not be decoded"} + } + + if !reflect.DeepEqual(oldLifecycleMetadata.EffectiveArgs, newLifecycleMetadata.EffectiveArgs) { + args = append(args, "Effective arguments changed") + } + + env = changedExecutableEnvNames(oldLifecycleMetadata.ExplicitEffectiveEnv, newLifecycleMetadata.ExplicitEffectiveEnv) + + if oldLifecycleMetadata.ExecutablePath != newLifecycleMetadata.ExecutablePath { + other = append(other, "Executable path changed") + } + if oldLifecycleMetadata.WorkingDirectory != newLifecycleMetadata.WorkingDirectory { + other = append(other, "Working directory changed") + } + if oldLifecycleMetadata.AmbientEnvironment != newLifecycleMetadata.AmbientEnvironment { + other = append(other, "Ambient environment behavior changed") + } + if !reflect.DeepEqual(oldLifecycleMetadata.PEMCertificates, newLifecycleMetadata.PEMCertificates) || + oldLifecycleMetadata.PEMCertificatesContinueOnError != newLifecycleMetadata.PEMCertificatesContinueOnError { + other = append(other, "Executable PEM certificates entries changed") + } + + return args, env, other +} + +func changedExecutableEnvNames(oldEnv, newEnv []persistentExecutableEnvMetadata) []string { + oldEnvByName := map[string]string{} + newEnvByName := map[string]string{} + for _, envVar := range oldEnv { + oldEnvByName[envVar.Name] = envVar.ValueHash + } + for _, envVar := range newEnv { + newEnvByName[envVar.Name] = envVar.ValueHash + } + + changedEnv := map[string]bool{} + for name, oldValueHash := range oldEnvByName { + if newValueHash, found := newEnvByName[name]; !found || oldValueHash != newValueHash { + changedEnv[name] = true + } + } + for name := range newEnvByName { + if _, found := oldEnvByName[name]; !found { + changedEnv[name] = true + } + } + + changedNames := maps.Keys(changedEnv) + stdslices.Sort(changedNames) + return changedNames +} + +func (r *ExecutableReconciler) deletePersistentProcessRecord(ctx context.Context, name types.NamespacedName, log logr.Logger) { + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + return + } + + deleteErr := stateStore.DeletePersistentProcess(ctx, name.String()) + if deleteErr != nil { + log.Error(deleteErr, "Could not delete persistent Executable process record", "Executable", name.String()) + } +} + +func (r *ExecutableReconciler) deletePersistentProcessRecordWithLease(ctx context.Context, name types.NamespacedName, log logr.Logger) { + exe := &apiv1.Executable{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: name.Namespace, + Name: name.Name, + }, + Spec: apiv1.ExecutableSpec{ + Persistent: true, + }, + } + _ = r.withPersistentExecutableLease(ctx, exe, log, func(ctx context.Context, _ *statestore.ResourceLease) objectChange { + r.deletePersistentProcessRecord(ctx, name, log) + return noChange + }) +} + +func (r *ExecutableReconciler) getStateStore() (*statestore.Store, error) { + if r.config.StateStore == nil { + return nil, fmt.Errorf("state store is not configured") + } + return r.config.StateStore, nil +} + +func (r *ExecutableReconciler) getResourceLeaseOwner() (process.ProcessTreeItem, error) { + if r.config.ResourceLeaseOwner.Pid > 0 && !r.config.ResourceLeaseOwner.IdentityTime.IsZero() { + return r.config.ResourceLeaseOwner, nil + } + return statestore.CurrentResourceLeaseOwner() +} + +func (r *ExecutableReconciler) releasePersistentExecutableResources(ctx context.Context, exe *apiv1.Executable, runInfo *ExecutableRunInfo, log logr.Logger) { + r.disableEndpointsAndHealthProbes(ctx, exe, runInfo, log) + logger.ReleaseResourceLog(exe.GetResourceId()) + if runInfo != nil && runInfo.RunID != UnknownRunID { + runner, runnerErr := r.getExecutableRunner(exe, runInfo.startupStage) + if runnerErr != nil { + log.Error(runnerErr, "Could not release persistent Executable runner tracking", "RunID", runInfo.RunID) + } else if releaseErr := runner.ReleaseRun(ctx, runInfo.RunID, log); releaseErr != nil { + log.Error(releaseErr, "Could not release persistent Executable runner tracking", "RunID", runInfo.RunID) + } + } + if r.persistentProcessRecordCanBeReused(ctx, exe, log) { + log.V(1).Info("Preserving persistent Executable process record for reuse", "ResourceKey", exe.GetLeaseKey()) + return + } + r.deletePersistentProcessRecord(ctx, exe.NamespacedName(), log) +} + +func (r *ExecutableReconciler) persistentProcessRecordCanBeReused(ctx context.Context, exe *apiv1.Executable, log logr.Logger) bool { + stateStore, stateStoreErr := r.getStateStore() + if stateStoreErr != nil { + log.Error(stateStoreErr, "Could not determine whether persistent Executable process record can be reused", "ResourceKey", exe.GetLeaseKey()) + return false + } + + record, recordErr := stateStore.GetPersistentProcess(ctx, exe.GetLeaseKey()) + if errors.Is(recordErr, statestore.ErrPersistentProcessNotFound) { + return false + } + if recordErr != nil { + log.Error(recordErr, "Could not read persistent Executable process record", "ResourceKey", exe.GetLeaseKey()) + return false + } + + lifecycleInfo, lifecycleKeyErr := persistentExecutableLifecycleInfo(exe) + if lifecycleKeyErr != nil { + log.Error(lifecycleKeyErr, "Could not calculate Executable lifecycle key") + return false + } + if record.LifecycleKey != lifecycleInfo.Key { + return false + } + + if _, findErr := process.FindProcess(record.PID, record.IdentityTime); findErr != nil { + log.Info("Persistent Executable process record is not reusable because the process is no longer running", + "PID", record.PID, + "Error", findErr.Error()) + return false + } + + return true +} diff --git a/controllers/executable_run_info.go b/controllers/executable_run_info.go index 8a7701e3..77a3efc0 100644 --- a/controllers/executable_run_info.go +++ b/controllers/executable_run_info.go @@ -8,6 +8,7 @@ package controllers import ( "fmt" stdlib_maps "maps" + "time" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,6 +74,9 @@ type ExecutableRunInfo struct { // Timestamp when the run was started StartupTimestamp metav1.MicroTime + // Raw process identity time used to protect against PID reuse. + ProcessIdentityTime time.Time + // Timestamp when the run was finished FinishTimestamp metav1.MicroTime @@ -171,6 +175,10 @@ func (ri *ExecutableRunInfo) UpdateFrom(other *ExecutableRunInfo) bool { } updated = setTimestampIfAfterOrUnknown(other.StartupTimestamp, &ri.StartupTimestamp) || updated + if !other.ProcessIdentityTime.IsZero() && !ri.ProcessIdentityTime.Equal(other.ProcessIdentityTime) { + ri.ProcessIdentityTime = other.ProcessIdentityTime + updated = true + } updated = setTimestampIfAfterOrUnknown(other.FinishTimestamp, &ri.FinishTimestamp) || updated if other.StdOutFile != "" && ri.StdOutFile != other.StdOutFile { @@ -257,6 +265,7 @@ func (ri *ExecutableRunInfo) Clone() *ExecutableRunInfo { retval.ReservedPorts = stdlib_maps.Clone(ri.ReservedPorts) } retval.StartupTimestamp = ri.StartupTimestamp + retval.ProcessIdentityTime = ri.ProcessIdentityTime retval.FinishTimestamp = ri.FinishTimestamp retval.StdOutFile = ri.StdOutFile retval.StdErrFile = ri.StdErrFile diff --git a/controllers/executable_runner.go b/controllers/executable_runner.go index 363d196f..1b11a5ff 100644 --- a/controllers/executable_runner.go +++ b/controllers/executable_runner.go @@ -7,6 +7,7 @@ package controllers import ( "context" + "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/types" @@ -30,7 +31,7 @@ type ExecutableRunner interface { // The runChangeHandler is used to notify the caller about the run's progress and completion, see RunChangeHandler for more details. // // The following contract should be observed by the ExecutableRunner implementation: - // -- When the passed context is cancelled, the run should be automatically terminated. + // -- When the passed context is cancelled, the run should be automatically terminated unless the Executable is persistent. // -- The runner should not try to change the passed Executable in any way. // -- The method should always return a result (no nil return value). StartRun( @@ -42,6 +43,30 @@ type ExecutableRunner interface { // Stops the run with a given ID. StopRun(ctx context.Context, runID RunID, log logr.Logger) error + + // Releases runner-side tracking for a run without stopping the underlying run. + ReleaseRun(ctx context.Context, runID RunID, log logr.Logger) error +} + +type ExecutableRunAdoptionInfo struct { + RunID RunID + Pid process.Pid_t + ProcessIdentityTime time.Time + StdOutFile string + StdErrFile string + CommandInfo string +} + +type PersistentExecutableRunner interface { + ExecutableRunner + + // Adopts a persistent run that was started by a previous DCP controller instance. + AdoptRun( + ctx context.Context, + run ExecutableRunAdoptionInfo, + runChangeHandler RunChangeHandler, + log logr.Logger, + ) error } type RunMessageLevel string diff --git a/controllers/executable_start_result.go b/controllers/executable_start_result.go index f209f100..77780119 100644 --- a/controllers/executable_start_result.go +++ b/controllers/executable_start_result.go @@ -7,6 +7,7 @@ package controllers import ( "fmt" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,6 +15,7 @@ import ( "github.com/microsoft/dcp/pkg/logger" "github.com/microsoft/dcp/pkg/osutil" "github.com/microsoft/dcp/pkg/pointers" + "github.com/microsoft/dcp/pkg/process" ) type ExecutableStartResult struct { @@ -38,6 +40,9 @@ type ExecutableStartResult struct { // Timestamp for when the startup attempt was completed CompletionTimestamp metav1.MicroTime + // Raw process identity time used to protect against PID reuse. + ProcessIdentityTime time.Time + // The error that occurred during startup, if any. // NOTE: if StartupError is nil, this does NOT necessarily mean that the startup was successful. // The ExeState field must be checked to determine the actual outcome of the startup attempt. @@ -94,6 +99,10 @@ func (res *ExecutableStartResult) Equal(other *ExecutableStartResult) bool { return false } + if !res.ProcessIdentityTime.Equal(other.ProcessIdentityTime) { + return false + } + // Check for error presence, not if errors are "equal" (which is not well defined). // We do not have a scenario where we set the StartupError more than once on a given result, // so this is sufficient. @@ -118,6 +127,7 @@ func (res *ExecutableStartResult) applyTo(ri *ExecutableRunInfo) { ri.StdOutFile = res.StdOutFile ri.StdErrFile = res.StdErrFile ri.StartupTimestamp = res.CompletionTimestamp + ri.ProcessIdentityTime = res.ProcessIdentityTime if res.ExeState == apiv1.ExecutableStateFinished { ri.FinishTimestamp = res.CompletionTimestamp } @@ -145,6 +155,7 @@ func (res *ExecutableStartResult) Clone() *ExecutableStartResult { StdOutFile: res.StdOutFile, StdErrFile: res.StdErrFile, CompletionTimestamp: res.CompletionTimestamp, + ProcessIdentityTime: res.ProcessIdentityTime, StartupError: res.StartupError, StartWaitForRunCompletion: res.StartWaitForRunCompletion, } @@ -193,6 +204,11 @@ func (res *ExecutableStartResult) UpdateFrom(other *ExecutableStartResult) bool updated = setTimestampIfAfterOrUnknown(other.CompletionTimestamp, &res.CompletionTimestamp) || updated + if !other.ProcessIdentityTime.IsZero() && !res.ProcessIdentityTime.Equal(other.ProcessIdentityTime) { + res.ProcessIdentityTime = other.ProcessIdentityTime + updated = true + } + if other.StartupError != nil && res.StartupError != other.StartupError { res.StartupError = other.StartupError updated = true @@ -211,7 +227,7 @@ func (res *ExecutableStartResult) String() string { return "{}" } - return fmt.Sprintf("{exeState: %s, pid: %v, runID: %s, exitCode: %s, stdOutFile: %s, stdErrFile: %s, startupCompletedTimestamp: %s, startupError: %s}", + return fmt.Sprintf("{exeState: %s, pid: %v, runID: %s, exitCode: %s, stdOutFile: %s, stdErrFile: %s, startupCompletedTimestamp: %s, processIdentityTime: %s, startupError: %s}", res.ExeState, logger.IntPtrValToString(res.Pid), logger.FriendlyString(string(res.RunID)), @@ -219,6 +235,7 @@ func (res *ExecutableStartResult) String() string { logger.FriendlyString(res.StdOutFile), logger.FriendlyString(res.StdErrFile), logger.FriendlyMetav1Timestamp(res.CompletionTimestamp), + logger.FriendlyString(process.FormatIdentityTime(res.ProcessIdentityTime)), logger.FriendlyErrorString(res.StartupError), ) } diff --git a/controllers/network_controller.go b/controllers/network_controller.go index b5dd90dd..aa6faf03 100644 --- a/controllers/network_controller.go +++ b/controllers/network_controller.go @@ -32,6 +32,7 @@ import ( apiv1 "github.com/microsoft/dcp/api/v1" "github.com/microsoft/dcp/internal/containers" "github.com/microsoft/dcp/internal/pubsub" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/pkg/commonapi" "github.com/microsoft/dcp/pkg/concurrency" "github.com/microsoft/dcp/pkg/osutil" @@ -132,6 +133,13 @@ type NetworkReconciler struct { // The resource harvester used to clean up abandoned resources on startup harvester *resourceHarvester + + config NetworkReconcilerConfig +} + +type NetworkReconcilerConfig struct { + StateStore *statestore.Store + ResourceLeaseOwner process.ProcessTreeItem } var ( @@ -145,6 +153,26 @@ func NewNetworkReconciler( log logr.Logger, orchestrator containers.ContainerOrchestrator, harvester *resourceHarvester, +) *NetworkReconciler { + return NewNetworkReconcilerWithConfig( + lifetimeCtx, + client, + noCacheClient, + log, + orchestrator, + harvester, + NetworkReconcilerConfig{}, + ) +} + +func NewNetworkReconcilerWithConfig( + lifetimeCtx context.Context, + client ctrl_client.Client, + noCacheClient ctrl_client.Reader, + log logr.Logger, + orchestrator containers.ContainerOrchestrator, + harvester *resourceHarvester, + config NetworkReconcilerConfig, ) *NetworkReconciler { base := NewReconcilerBase[apiv1.ContainerNetwork](client, noCacheClient, log, lifetimeCtx) @@ -159,6 +187,7 @@ func NewNetworkReconciler( lock: &sync.Mutex{}, orchestratorHealthy: &atomic.Bool{}, harvester: harvester, + config: config, } go r.onShutdown() @@ -335,9 +364,7 @@ func (r *NetworkReconciler) manageNetwork(ctx context.Context, network *apiv1.Co } if network.Status.State != networkState.state { - network.Status.State = networkState.state - network.Status.Message = networkState.message - change |= statusChanged + change |= r.setNetworkState(network, networkState.state, networkState.message) } if network.Status.State == apiv1.ContainerNetworkStateRunning { @@ -353,7 +380,56 @@ func (r *NetworkReconciler) ensureNetwork(ctx context.Context, network *apiv1.Co r.ensureNetworkWatch(network, log) networkName := strings.TrimSpace(network.Spec.NetworkName) + if network.Spec.Persistent { + var change objectChange + if r.config.StateStore == nil { + stateStoreErr := fmt.Errorf("state store is not configured") + log.Error(stateStoreErr, "Could not acquire persistent network lease") + return r.failNetworkToStart(network, networkName, stateStoreErr) + } + + leaseErr := r.config.StateStore.WithResourceLease( + ctx, + network, + r.config.ResourceLeaseOwner, + resourceLeaseRevalidationInterval, + func(_ context.Context, lease *statestore.ResourceLease) error { + log.V(1).Info("Acquired resource lease", "ResourceKey", lease.ResourceKey) + change = r.ensureNetworkWithName(ctx, network, networkName, log) + return nil + }, + ) + if errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + logResourceLeaseHeld(log, leaseErr, network.GetLeaseKey(), "Persistent network is being updated by another DCP instance, retrying") + return additionalReconciliationNeeded + } + if leaseErr != nil { + log.Error(leaseErr, "Could not acquire persistent network lease") + return r.failNetworkToStart(network, networkName, leaseErr) + } + + if persistentNetworkLeaseReleaseState(network.Status.State) { + // Intentionally ignore errors: this is a defensive, idempotent release attempt for stable states. + _ = r.releasePersistentNetworkResourceLease(context.Background(), network, log, true) + } + + return change + } + return r.ensureNetworkWithName(ctx, network, networkName, log) +} + +func (r *NetworkReconciler) failNetworkToStart(network *apiv1.ContainerNetwork, networkName string, err error) objectChange { + if r.existingNetworks != nil { + r.existingNetworks.Store(network.NamespacedName(), networkName, &runningNetworkState{ + state: apiv1.ContainerNetworkStateFailedToStart, + message: err.Error(), + }) + } + return r.setNetworkState(network, apiv1.ContainerNetworkStateFailedToStart, err.Error()) +} + +func (r *NetworkReconciler) ensureNetworkWithName(ctx context.Context, network *apiv1.ContainerNetwork, networkName string, log logr.Logger) objectChange { if network.Spec.Persistent { isNetworkSafeToReuse := r.harvester.IsDone() || r.harvester.TryProtectNetwork(ctx, networkName) existing, err := inspectNetworkIfExists(ctx, r.orchestrator, networkName) @@ -582,9 +658,8 @@ func (r *NetworkReconciler) ensureConnections(ctx context.Context, network *apiv func (r *NetworkReconciler) updateNetworkStatus(ctx context.Context, network *apiv1.ContainerNetwork, rns *runningNetworkState, log logr.Logger) objectChange { cnet, err := inspectNetwork(ctx, r.orchestrator, network.Status.ID) if errors.Is(err, containers.ErrNotFound) { - network.Status.State = apiv1.ContainerNetworkStateRemoved rns.state = apiv1.ContainerNetworkStateRemoved - return statusChanged + return r.setNetworkState(network, apiv1.ContainerNetworkStateRemoved, network.Status.Message) } else if err != nil { log.Error(err, "Could not inspect a network") return additionalReconciliationNeeded @@ -625,6 +700,62 @@ func (r *NetworkReconciler) updateNetworkStatus(ctx context.Context, network *ap return change } +func (r *NetworkReconciler) setNetworkState(network *apiv1.ContainerNetwork, state apiv1.ContainerNetworkState, message string) objectChange { + change := noChange + + if network.Status.State != state { + network.Status.State = state + change = statusChanged + } + if network.Status.Message != message { + network.Status.Message = message + change = statusChanged + } + + if network.Spec.Persistent && persistentNetworkLeaseReleaseState(state) { + // Intentionally ignore errors: this is a defensive, idempotent release attempt for stable states. + _ = r.releasePersistentNetworkResourceLease(context.Background(), network, r.Log, true) + } + + return change +} + +func persistentNetworkLeaseReleaseState(state apiv1.ContainerNetworkState) bool { + switch state { + case apiv1.ContainerNetworkStateRunning, + apiv1.ContainerNetworkStateFailedToStart, + apiv1.ContainerNetworkStateRemoved, + apiv1.ContainerNetworkStateNotFound: + return true + default: + return false + } +} + +func (r *NetworkReconciler) releasePersistentNetworkResourceLease( + ctx context.Context, + network *apiv1.ContainerNetwork, + log logr.Logger, + suppressNotHeldLog bool, +) error { + if !network.Spec.Persistent || r.config.StateStore == nil || + r.config.ResourceLeaseOwner.Pid <= 0 || r.config.ResourceLeaseOwner.IdentityTime.IsZero() { + return nil + } + + releaseErr := r.config.StateStore.ReleaseResourceLease(ctx, network, r.config.ResourceLeaseOwner) + if releaseErr != nil { + if errors.Is(releaseErr, statestore.ErrResourceLeaseNotHeld) { + logResourceLeaseNotHeld(log, suppressNotHeldLog, network.GetLeaseKey(), "Persistent network resource lease was not held") + return releaseErr + } + log.Error(releaseErr, "Could not release persistent network resource lease") + return releaseErr + } + + return nil +} + func (r *NetworkReconciler) ensureNetworkWatch(network *apiv1.ContainerNetwork, log logr.Logger) { r.lock.Lock() defer r.lock.Unlock() diff --git a/controllers/resource_lease_logging.go b/controllers/resource_lease_logging.go new file mode 100644 index 00000000..f1e4657c --- /dev/null +++ b/controllers/resource_lease_logging.go @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package controllers + +import ( + "github.com/go-logr/logr" + + "github.com/microsoft/dcp/internal/statestore" +) + +func logResourceLeaseHeld(log logr.Logger, leaseErr error, resourceKey string, message string) { + logValues := []any{"ResourceKey", resourceKey} + if heldLease, found := statestore.HeldResourceLease(leaseErr); found { + logValues = []any{ + "ResourceKey", heldLease.ResourceKey, + "LeaseOwnerPID", heldLease.OwnerProcess.Pid, + "LeaseOwnerIdentityTime", heldLease.OwnerProcess.IdentityTime, + } + } + + log.V(1).Info(message, logValues...) +} + +func logResourceLeaseNotHeld(log logr.Logger, suppress bool, resourceKey string, message string) { + if suppress { + return + } + log.V(1).Info(message, "ResourceKey", resourceKey) +} diff --git a/controllers/service_controller_metrics.go b/controllers/service_controller_metrics.go index 3f1a3c22..d62e2510 100644 --- a/controllers/service_controller_metrics.go +++ b/controllers/service_controller_metrics.go @@ -3,88 +3,87 @@ * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ - -package controllers - -import ( - "context" - - "go.opentelemetry.io/otel/metric" - "golang.org/x/net/nettest" - - apiv1 "github.com/microsoft/dcp/api/v1" - "github.com/microsoft/dcp/internal/telemetry" -) - -var ( - localhostAllocationModeCounter metric.Int64UpDownCounter - ipv4ZeroOneAllocationModeCounter metric.Int64UpDownCounter - ipv4LoopbackAllocationModeCounter metric.Int64UpDownCounter - ipv6ZeroOneAllocationModeCounter metric.Int64UpDownCounter - proxylessAllocationModeCounter metric.Int64UpDownCounter - - ipv4OnlyCounter metric.Int64UpDownCounter - ipv6OnlyCounter metric.Int64UpDownCounter - dualStackCounter metric.Int64UpDownCounter - - tcpServiceCounter metric.Int64UpDownCounter - udpServiceCounter metric.Int64UpDownCounter -) - -func init() { - ts := telemetry.GetTelemetrySystem() - svcCtrlMeter := ts.MeterProvider.Meter("service-controller") - - localhostAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "localhostAllocationModeServices", "Number of services that use the localhost address allocation mode") - ipv4ZeroOneAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv4ZeroOneAllocationModeServices", "Number of services that use the IPv4 127.0.0.1 address allocation mode") - ipv4LoopbackAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv4LoopbackAllocationModeServices", "Number of services that use the IPv4 random loopback address allocation mode") - ipv6ZeroOneAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv6ZeroOneAllocationModeServices", "Number of services that use the IPv6 [::1] address allocation mode") - proxylessAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "proxylessAllocationModeServices", "Number of services that use the proxyless address allocation mode") - - ipv4OnlyCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv4OnlyServices", "Number of services that only use IPv4") - ipv6OnlyCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv6OnlyServices", "Number of services that only use IPv6") - dualStackCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "dualStackServices", "Number of services that use both IPv4 and IPv6") - - tcpServiceCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "tcpServices", "Number of services that use TCP") - udpServiceCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "udpServices", "Number of services that use UDP") -} - -func serviceCounters(ctx context.Context, service *apiv1.Service, addend int64) { - switch service.Spec.AddressAllocationMode { - case apiv1.AddressAllocationModeLocalhost: - localhostAllocationModeCounter.Add(ctx, addend) - stackCounters(ctx, addend) - case apiv1.AddressAllocationModeIPv4ZeroOne: - ipv4ZeroOneAllocationModeCounter.Add(ctx, addend) - ipv4OnlyCounter.Add(ctx, addend) - case apiv1.AddressAllocationModeIPv4Loopback: - ipv4LoopbackAllocationModeCounter.Add(ctx, addend) - ipv4OnlyCounter.Add(ctx, addend) - case apiv1.AddressAllocationModeIPv6ZeroOne: - ipv6ZeroOneAllocationModeCounter.Add(ctx, addend) - ipv6OnlyCounter.Add(ctx, addend) - case apiv1.AddressAllocationModeProxyless: - proxylessAllocationModeCounter.Add(ctx, addend) - } - - switch service.Spec.Protocol { - case apiv1.TCP: - tcpServiceCounter.Add(ctx, addend) - case apiv1.UDP: - udpServiceCounter.Add(ctx, addend) - } -} - -func stackCounters(ctx context.Context, addend int64) { - // Localhost can be both IPv4 and IPv6, check to see if the machine supports IPv6 - supportsIPv4 := nettest.SupportsIPv4() - supportsIPv6 := nettest.SupportsIPv6() - - if supportsIPv4 && supportsIPv6 { - dualStackCounter.Add(ctx, addend) - } else if supportsIPv4 { - ipv4OnlyCounter.Add(ctx, addend) - } else if supportsIPv6 { - ipv6OnlyCounter.Add(ctx, addend) - } -} +package controllers + +import ( + "context" + + "go.opentelemetry.io/otel/metric" + "golang.org/x/net/nettest" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/telemetry" +) + +var ( + localhostAllocationModeCounter metric.Int64UpDownCounter + ipv4ZeroOneAllocationModeCounter metric.Int64UpDownCounter + ipv4LoopbackAllocationModeCounter metric.Int64UpDownCounter + ipv6ZeroOneAllocationModeCounter metric.Int64UpDownCounter + proxylessAllocationModeCounter metric.Int64UpDownCounter + + ipv4OnlyCounter metric.Int64UpDownCounter + ipv6OnlyCounter metric.Int64UpDownCounter + dualStackCounter metric.Int64UpDownCounter + + tcpServiceCounter metric.Int64UpDownCounter + udpServiceCounter metric.Int64UpDownCounter +) + +func init() { + ts := telemetry.GetTelemetrySystem() + svcCtrlMeter := ts.MeterProvider.Meter("service-controller") + + localhostAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "localhostAllocationModeServices", "Number of services that use the localhost address allocation mode") + ipv4ZeroOneAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv4ZeroOneAllocationModeServices", "Number of services that use the IPv4 127.0.0.1 address allocation mode") + ipv4LoopbackAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv4LoopbackAllocationModeServices", "Number of services that use the IPv4 random loopback address allocation mode") + ipv6ZeroOneAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv6ZeroOneAllocationModeServices", "Number of services that use the IPv6 [::1] address allocation mode") + proxylessAllocationModeCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "proxylessAllocationModeServices", "Number of services that use the proxyless address allocation mode") + + ipv4OnlyCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv4OnlyServices", "Number of services that only use IPv4") + ipv6OnlyCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "ipv6OnlyServices", "Number of services that only use IPv6") + dualStackCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "dualStackServices", "Number of services that use both IPv4 and IPv6") + + tcpServiceCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "tcpServices", "Number of services that use TCP") + udpServiceCounter = telemetry.NewInt64UpDownCounter(svcCtrlMeter, "udpServices", "Number of services that use UDP") +} + +func serviceCounters(ctx context.Context, service *apiv1.Service, addend int64) { + switch service.Spec.AddressAllocationMode { + case apiv1.AddressAllocationModeLocalhost: + localhostAllocationModeCounter.Add(ctx, addend) + stackCounters(ctx, addend) + case apiv1.AddressAllocationModeIPv4ZeroOne: + ipv4ZeroOneAllocationModeCounter.Add(ctx, addend) + ipv4OnlyCounter.Add(ctx, addend) + case apiv1.AddressAllocationModeIPv4Loopback: + ipv4LoopbackAllocationModeCounter.Add(ctx, addend) + ipv4OnlyCounter.Add(ctx, addend) + case apiv1.AddressAllocationModeIPv6ZeroOne: + ipv6ZeroOneAllocationModeCounter.Add(ctx, addend) + ipv6OnlyCounter.Add(ctx, addend) + case apiv1.AddressAllocationModeProxyless: + proxylessAllocationModeCounter.Add(ctx, addend) + } + + switch service.Spec.Protocol { + case apiv1.TCP: + tcpServiceCounter.Add(ctx, addend) + case apiv1.UDP: + udpServiceCounter.Add(ctx, addend) + } +} + +func stackCounters(ctx context.Context, addend int64) { + // Localhost can be both IPv4 and IPv6, check to see if the machine supports IPv6 + supportsIPv4 := nettest.SupportsIPv4() + supportsIPv6 := nettest.SupportsIPv6() + + if supportsIPv4 && supportsIPv6 { + dualStackCounter.Add(ctx, addend) + } else if supportsIPv4 { + ipv4OnlyCounter.Add(ctx, addend) + } else if supportsIPv6 { + ipv6OnlyCounter.Add(ctx, addend) + } +} diff --git a/doc/executable-persistence-api.md b/doc/executable-persistence-api.md new file mode 100644 index 00000000..638f43d6 --- /dev/null +++ b/doc/executable-persistence-api.md @@ -0,0 +1,152 @@ +# Executable persistence + +Executable resources can be configured so their underlying OS process survives a DCP controller restart and can be adopted by the next controller instance. + +## `ExecutableSpec` fields + +### `spec.start` + +`start` is an optional boolean that controls whether the controller should attempt to start the Executable. + +```yaml +spec: + start: true +``` + +Behavior: + +- Omitted means `true`. +- `false` is only useful on initial creation. It creates the Executable object without starting a process. +- Once an Executable has been created with `start` omitted or set to `true`, it cannot be updated to `false`. +- `start` does not replace `stop`; use `spec.stop: true` to request that a started Executable be stopped. + +Use this when a caller needs to define the Executable object before allowing DCP to start it. + +### `spec.persistent` + +`persistent` is an optional boolean that marks the Executable's process as persistent across DCP runs. + +```yaml +spec: + persistent: true +``` + +Behavior: + +- Omitted means `false`; existing Executables keep their previous non-persistent behavior. +- Persistent Executables are supported only with `executionType: Process`, or with `executionType` omitted because `Process` is the default. +- Persistent Executables cannot specify `fallbackExecutionTypes`. +- `persistent` is immutable after creation. +- Deleting a persistent Executable object releases DCP's ownership/lease metadata but intentionally leaves the underlying process running. +- Setting `spec.stop: true` still requests that DCP stop the underlying process. + +When a persistent Executable starts successfully, DCP stores process identity metadata in the local SQLite state store. A later DCP controller instance can use that record to reattach status, logs, endpoints, and health probe handling to the already-running process instead of launching a duplicate process. + +### `spec.lifecycleKey` + +`lifecycleKey` is an optional string that identifies whether an existing persistent process record is reusable for a newly reconciled Executable. + +```yaml +spec: + persistent: true + lifecycleKey: my-service-dev-loop +``` + +Behavior: + +- If set, DCP uses this value directly. +- If omitted, DCP computes a lifecycle key from the launch-relevant parts of the Executable spec: + - `executablePath` + - `workingDirectory` + - `executionType` + - `ambientEnvironment.behavior` + - effective `args` after template substitution + - effective values for environment variables explicitly listed in `env` + - effective values for environment variables loaded from `envFiles` + - `pemCertificates` +- Inherited/system environment variables are ignored unless they are also explicitly listed in `env` or loaded from `envFiles`. +- When DCP finds a stored persistent process record, it adopts the process only if the stored lifecycle key matches the current lifecycle key. +- If the key does not match, DCP logs the likely source of the change for default lifecycle keys, deletes the stale process record, and starts a new process. + +For default lifecycle keys, the persistent process record also stores lifecycle diagnostic metadata. On mismatch, DCP logs argument, environment, and other change categories. Environment diagnostics identify changed variable names and store value hashes rather than raw environment values. + +Use an explicit `lifecycleKey` when the caller wants to control process reuse across object recreation, or when the default hash would be too sensitive to inputs that should not force a restart. + +## Example persistent Executable + +```yaml +apiVersion: usvc.dev/v1 +kind: Executable +metadata: + name: my-service +spec: + executablePath: /path/to/my-service + workingDirectory: /path/to/repo + args: + - --urls + - http://127.0.0.1:5000 + executionType: Process + persistent: true + lifecycleKey: my-service +``` + +With this configuration: + +1. The first DCP controller instance starts `/path/to/my-service`. +2. DCP records the process PID, process identity time, run ID, log file paths, execution type, lifecycle key, and lifecycle diagnostic metadata in the local state store. +3. If the controller exits and a new one starts, it reads the stored record. +4. If the process is still running and the lifecycle key still matches, DCP adopts that process instead of starting another copy. +5. If the process is gone or the lifecycle key changed, DCP removes the stale record and starts a new process. + +## State store behavior + +The backing store is a local SQLite database used for DCP coordination metadata, including persistent process records and resource leases. + +- The default path is under the DCP per-user directory. +- `DCP_STATE_STORE_PATH` overrides the SQLite database path. +- Elevated/admin DCP processes use a separate default database name from non-elevated processes. +- DCP uses WAL mode and a busy timeout to coordinate concurrent controller processes. +- Migration SQL is embedded into the DCP binary, so runtime schema initialization does not depend on external migration files. + +The schema includes: + +- `persistent_processes`: process records used to adopt persistent Executables. +- `resource_locks`: lease records used for coordination between DCP processes. + +## Go behavior + +The following behavior is relevant to code that constructs Executable objects, implements Executable runners, or creates controller reconcilers in tests. + +### Executable construction and comparison + +Code that constructs Executable specs can opt into three behaviors: deferring initial start, making the run persistent, and controlling the lifecycle key used for reuse. Spec comparisons use effective start behavior, so omitted `start` and explicit `true` are treated as equivalent. + +When callers omit `lifecycleKey`, the controller computes one from launch-relevant inputs. That computation can depend on resolved arguments, explicitly configured environment variables, environment files, and certificate configuration, so tests that exercise default lifecycle-key behavior need those referenced inputs to be available. + +### Runner responsibilities + +Executable runners distinguish between ordinary controller-owned runs and persistent runs. For ordinary runs, cancellation of the controller lifetime should still clean up the process. For persistent runs, the runner must let the process outlive the controller instance and must avoid any monitor or process flags that would kill the process when DCP exits. + +Runners that support persistence also need an adoption path. Adoption reattaches DCP bookkeeping to a process started by an earlier controller instance rather than launching a new process. Implementations should validate the stored run metadata, verify the process by both PID and process identity time to protect against PID reuse, and reconnect the run to the normal completion/stop notification flow. + +Runner results distinguish between the process identity timestamp and the display start timestamp. The identity timestamp is for correctness checks and persistence; the display timestamp is for user-facing status. Persistent runs must provide enough process identity information for the controller to safely adopt or reject a stored record later. + +### Reconciler and state-store behavior + +Tests can override the state store location when they need isolated persistence state. + +Persistent Executable records are keyed by the Executable's namespaced name because the persistent-process table already scopes the record type. Runtime resource leases for containers and networks use a separate lease-key interface because those leases coordinate external Docker/Podman resources in a shared table; those keys are based on runtime resource names rather than Kubernetes object names. + +The state store owns the details of persisting process identity, lifecycle metadata, and resource lease ownership. Callers should treat stored records as controller coordination data, not as a stable public serialization contract. + +## Configuration checklist + +When configuring persistent Executables: + +1. Add `spec.persistent: true` only for Executables that should keep running when DCP exits or restarts. +2. Keep persistent Executables on `executionType: Process`; do not use IDE execution or fallback execution types with persistence. +3. Decide whether the default lifecycle-key hash is appropriate. If stable caller-controlled reuse is needed, set `spec.lifecycleKey`. +4. Do not update an existing Executable from non-persistent to persistent, or from persistent to non-persistent. Recreate the object instead. +5. Do not set `spec.start: false` on an Executable that was already created with start enabled. Recreate the object if a not-yet-started resource is required. +6. Use `spec.stop: true` when the persistent process should be stopped; deleting the Executable object alone leaves the process running. +7. If tests need isolation from a developer's real state store, set `DCP_STATE_STORE_PATH` to a temporary SQLite file. diff --git a/go.mod b/go.mod index 8ee60bc4..34e31cc6 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/felixge/fgprof v0.9.5 github.com/go-logr/logr v1.4.3 github.com/go-logr/zapr v1.3.0 + github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 @@ -38,6 +39,7 @@ require ( k8s.io/client-go v0.36.0 k8s.io/klog/v2 v2.140.0 k8s.io/kube-openapi v0.0.0-20260502001324-b7f5293f4787 + modernc.org/sqlite v1.50.1 sigs.k8s.io/controller-runtime v0.24.0 ) @@ -103,6 +105,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/ettle/strcase v0.2.0 // indirect @@ -216,6 +219,7 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nakabonne/nestif v0.3.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect github.com/nunnatsa/ginkgolinter v0.23.0 // indirect @@ -233,6 +237,7 @@ require ( github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/raeperd/recvcheck v0.2.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/ryancurrah/gomodguard v1.4.1 // indirect @@ -312,6 +317,9 @@ require ( k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect k8s.io/streaming v0.36.0 // indirect k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 // indirect + modernc.org/libc v1.72.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect diff --git a/go.sum b/go.sum index 8c082b69..35c48817 100644 --- a/go.sum +++ b/go.sum @@ -262,6 +262,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= +github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= @@ -398,6 +400,8 @@ github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= +github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= +github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -445,6 +449,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= @@ -498,6 +504,8 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4l github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/raeperd/recvcheck v0.2.0 h1:GnU+NsbiCqdC2XX5+vMZzP+jAJC5fht7rcVTAhX74UI= github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -863,6 +871,34 @@ k8s.io/streaming v0.36.0 h1:agnTxU+NFulUrtYzXUGKO3ndEa8jKwht1Kwn9nu9x+4= k8s.io/streaming v0.36.0/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.50.1 h1:l+cQvn0sd0zJJtfygGHuQJ5AjlrwXmWPw4KP3ZMwr9w= +modernc.org/sqlite v1.50.1/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 h1:ssMzja7PDPJV8FStj7hq9IKiuiKhgz9ErWw+m68e7DI= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 9b3ee6c7..dc58d75f 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ diff --git a/internal/apiserver/admin_http_handler.go b/internal/apiserver/admin_http_handler.go index 3db49a9e..fbea0ba8 100644 --- a/internal/apiserver/admin_http_handler.go +++ b/internal/apiserver/admin_http_handler.go @@ -212,20 +212,23 @@ func (h *adminHttpHandler) changeExecution(w http.ResponseWriter, r *http.Reques return } + w.Header().Set("Content-Type", "application/json") if changedStatus { w.WriteHeader(http.StatusAccepted) // Accepted means operation has been started } else { w.WriteHeader(http.StatusOK) // OK means operation is in progress or has already been completed } - w.Header().Set("Content-Type", "application/json") _, writeErr := w.Write(resp) if writeErr != nil { h.log.Error(writeErr, "Could not write API server execution data") } - // Only request shutdown AFTER writing the response, so that we do not "cancel ourselves" in the middle of writing. + // Only request shutdown AFTER flushing the response, so that we do not "cancel ourselves" in the middle of writing. if changedStatus && newStatus == ApiServerStopping { + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } h.runConfig.RequestShutdown(h.executionData.ShutdownResourceCleanup) } } diff --git a/internal/dcpctrl/commands/persistent_process_cleanup.go b/internal/dcpctrl/commands/persistent_process_cleanup.go new file mode 100644 index 00000000..6f97dfca --- /dev/null +++ b/internal/dcpctrl/commands/persistent_process_cleanup.go @@ -0,0 +1,147 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package commands + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/go-logr/logr" + + "github.com/microsoft/dcp/internal/logs" + "github.com/microsoft/dcp/internal/statestore" + "github.com/microsoft/dcp/pkg/process" + "github.com/microsoft/dcp/pkg/resiliency" +) + +const persistentProcessCleanupLeaseRevalidationInterval = 30 * time.Second + +type persistentProcessLeaseResource string + +func (r persistentProcessLeaseResource) GetLeaseKey() string { + return string(r) +} + +func startInvalidPersistentExecutableRecordCleanup( + ctx context.Context, + stateStore *statestore.Store, + leaseOwner process.ProcessTreeItem, + log logr.Logger, +) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + defer func() { _ = resiliency.MakePanicError(recover(), log) }() + + cleanupErr := cleanupInvalidPersistentExecutableRecords(ctx, stateStore, leaseOwner, log) + if cleanupErr != nil { + log.Error(cleanupErr, "Failed to clean up invalid persistent Executable process records") + } + }() + return done +} + +func cleanupInvalidPersistentExecutableRecords( + ctx context.Context, + stateStore *statestore.Store, + leaseOwner process.ProcessTreeItem, + log logr.Logger, +) error { + if stateStore == nil { + return fmt.Errorf("state store is not configured") + } + + records, listErr := stateStore.ListPersistentProcesses(ctx) + if listErr != nil { + return listErr + } + + var cleanupErr error + for _, record := range records { + if _, findErr := process.FindProcess(record.PID, record.IdentityTime); findErr == nil { + continue + } + + recordCleanupErr := cleanupInvalidPersistentExecutableRecord(ctx, stateStore, leaseOwner, record, log) + if recordCleanupErr != nil { + cleanupErr = errors.Join(cleanupErr, recordCleanupErr) + } + } + + return cleanupErr +} + +func cleanupInvalidPersistentExecutableRecord( + ctx context.Context, + stateStore *statestore.Store, + leaseOwner process.ProcessTreeItem, + record statestore.PersistentProcessRecord, + log logr.Logger, +) error { + resource := persistentProcessLeaseResource(record.ResourceKey) + leaseErr := stateStore.WithResourceLease(ctx, resource, leaseOwner, persistentProcessCleanupLeaseRevalidationInterval, func(ctx context.Context, _ *statestore.ResourceLease) error { + currentRecord, getErr := stateStore.GetPersistentProcess(ctx, record.ResourceKey) + if errors.Is(getErr, statestore.ErrPersistentProcessNotFound) { + log.V(1).Info("Persistent Executable process record was removed before cleanup, leaving it intact", + "ResourceKey", record.ResourceKey) + return nil + } + if getErr != nil { + return fmt.Errorf("could not reload persistent Executable process record '%s': %w", record.ResourceKey, getErr) + } + + _, findErr := process.FindProcess(currentRecord.PID, currentRecord.IdentityTime) + if findErr == nil { + log.V(1).Info("Persistent Executable process record became valid before cleanup, leaving it intact", + "ResourceKey", currentRecord.ResourceKey, + "PID", currentRecord.PID) + return nil + } + + log.Info("Deleting invalid persistent Executable process record", + "ResourceKey", currentRecord.ResourceKey, + "PID", currentRecord.PID, + "Error", findErr.Error()) + + if deleteErr := stateStore.DeletePersistentProcess(ctx, currentRecord.ResourceKey); deleteErr != nil { + return fmt.Errorf("could not delete invalid persistent Executable process record '%s': %w", currentRecord.ResourceKey, deleteErr) + } + return removePersistentExecutableRecordLogs(ctx, *currentRecord, log) + }) + if errors.Is(leaseErr, statestore.ErrResourceLeaseHeld) { + if lease, held := statestore.HeldResourceLease(leaseErr); held { + log.V(1).Info("Invalid persistent Executable process record cleanup lease is held by another DCP instance, leaving it intact", + "ResourceKey", record.ResourceKey, + "LeaseOwnerPID", lease.OwnerProcess.Pid) + } + return nil + } + if leaseErr != nil { + return fmt.Errorf("could not clean up invalid persistent Executable process record '%s': %w", record.ResourceKey, leaseErr) + } + + return nil +} + +func removePersistentExecutableRecordLogs(ctx context.Context, record statestore.PersistentProcessRecord, log logr.Logger) error { + removeLog := func(path string, stream string) error { + if path == "" { + return nil + } + if removeErr := logs.RemoveWithRetry(ctx, path); removeErr != nil { + log.Error(removeErr, "Could not remove persistent Executable log file", "ResourceKey", record.ResourceKey, "Path", path, "Stream", stream) + return removeErr + } + return nil + } + + return errors.Join( + removeLog(record.StdOutFile, "stdout"), + removeLog(record.StdErrFile, "stderr"), + ) +} diff --git a/internal/dcpctrl/commands/persistent_process_cleanup_test.go b/internal/dcpctrl/commands/persistent_process_cleanup_test.go new file mode 100644 index 00000000..52a4d083 --- /dev/null +++ b/internal/dcpctrl/commands/persistent_process_cleanup_test.go @@ -0,0 +1,255 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package commands + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + + "github.com/microsoft/dcp/internal/statestore" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/process" + "github.com/microsoft/dcp/pkg/testutil" +) + +func TestCleanupInvalidPersistentExecutableRecordsDeletesInvalidRecordsAndLogs(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + stateStore := openPersistentProcessCleanupTestStore(t, ctx) + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + require.NoError(t, leaseOwnerErr) + + logDir := t.TempDir() + activeStdout := createPersistentProcessCleanupLog(t, logDir, "active.out") + activeStderr := createPersistentProcessCleanupLog(t, logDir, "active.err") + invalidStdout := createPersistentProcessCleanupLog(t, logDir, "invalid.out") + invalidStderr := createPersistentProcessCleanupLog(t, logDir, "invalid.err") + + activeRecord := statestore.PersistentProcessRecord{ + ResourceKey: "default/active", + LifecycleKey: "active-lifecycle", + PID: leaseOwner.Pid, + IdentityTime: leaseOwner.IdentityTime, + RunID: "active", + StdOutFile: activeStdout, + StdErrFile: activeStderr, + } + invalidRecord := statestore.PersistentProcessRecord{ + ResourceKey: "default/invalid", + LifecycleKey: "invalid-lifecycle", + PID: leaseOwner.Pid, + IdentityTime: time.Unix(1, 0).UTC(), + RunID: "invalid", + StdOutFile: invalidStdout, + StdErrFile: invalidStderr, + } + require.NoError(t, stateStore.UpsertPersistentProcess(ctx, activeRecord)) + require.NoError(t, stateStore.UpsertPersistentProcess(ctx, invalidRecord)) + + require.NoError(t, cleanupInvalidPersistentExecutableRecords(ctx, stateStore, leaseOwner, logr.Discard())) + + _, invalidGetErr := stateStore.GetPersistentProcess(ctx, invalidRecord.ResourceKey) + require.ErrorIs(t, invalidGetErr, statestore.ErrPersistentProcessNotFound) + _, activeGetErr := stateStore.GetPersistentProcess(ctx, activeRecord.ResourceKey) + require.NoError(t, activeGetErr) + require.NoFileExists(t, invalidStdout) + require.NoFileExists(t, invalidStderr) + require.FileExists(t, activeStdout) + require.FileExists(t, activeStderr) +} + +func TestStartInvalidPersistentExecutableRecordCleanupRunsCleanupAsync(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + stateStore := openPersistentProcessCleanupTestStore(t, ctx) + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + require.NoError(t, leaseOwnerErr) + + logDir := t.TempDir() + invalidStdout := createPersistentProcessCleanupLog(t, logDir, "async-invalid.out") + invalidStderr := createPersistentProcessCleanupLog(t, logDir, "async-invalid.err") + + invalidRecord := statestore.PersistentProcessRecord{ + ResourceKey: "default/async-invalid", + LifecycleKey: "async-invalid-lifecycle", + PID: leaseOwner.Pid, + IdentityTime: time.Unix(1, 0).UTC(), + RunID: "async-invalid", + StdOutFile: invalidStdout, + StdErrFile: invalidStderr, + } + require.NoError(t, stateStore.UpsertPersistentProcess(ctx, invalidRecord)) + + done := startInvalidPersistentExecutableRecordCleanup(ctx, stateStore, leaseOwner, logr.Discard()) + select { + case <-done: + case <-ctx.Done(): + require.NoError(t, ctx.Err()) + } + + _, invalidGetErr := stateStore.GetPersistentProcess(ctx, invalidRecord.ResourceKey) + require.ErrorIs(t, invalidGetErr, statestore.ErrPersistentProcessNotFound) + require.NoFileExists(t, invalidStdout) + require.NoFileExists(t, invalidStderr) +} + +func TestCleanupInvalidPersistentExecutableRecordsSkipsRecordsWithHeldLease(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + stateStore := openPersistentProcessCleanupTestStore(t, ctx) + heldLeaseOwner, heldLeaseOwnerErr := statestore.CurrentResourceLeaseOwner() + require.NoError(t, heldLeaseOwnerErr) + cleanupLeaseOwner := process.ProcessTreeItem{ + Pid: heldLeaseOwner.Pid, + IdentityTime: heldLeaseOwner.IdentityTime.Add(-time.Hour), + } + + logDir := t.TempDir() + heldStdout := createPersistentProcessCleanupLog(t, logDir, "held.out") + heldStderr := createPersistentProcessCleanupLog(t, logDir, "held.err") + + heldRecord := statestore.PersistentProcessRecord{ + ResourceKey: "default/held", + LifecycleKey: "held-lifecycle", + PID: heldLeaseOwner.Pid, + IdentityTime: time.Unix(1, 0).UTC(), + RunID: "held", + StdOutFile: heldStdout, + StdErrFile: heldStderr, + } + require.NoError(t, stateStore.UpsertPersistentProcess(ctx, heldRecord)) + _, leaseErr := stateStore.AcquireResourceLease(ctx, persistentProcessLeaseResource(heldRecord.ResourceKey), heldLeaseOwner, time.Minute) + require.NoError(t, leaseErr) + + require.NoError(t, cleanupInvalidPersistentExecutableRecords(ctx, stateStore, cleanupLeaseOwner, logr.Discard())) + + _, getErr := stateStore.GetPersistentProcess(ctx, heldRecord.ResourceKey) + require.NoError(t, getErr) + require.FileExists(t, heldStdout) + require.FileExists(t, heldStderr) +} + +func TestCleanupInvalidPersistentExecutableRecordReloadsRecordAfterAcquiringLease(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + stateStore := openPersistentProcessCleanupTestStore(t, ctx) + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + require.NoError(t, leaseOwnerErr) + + logDir := t.TempDir() + staleStdout := createPersistentProcessCleanupLog(t, logDir, "stale.out") + staleStderr := createPersistentProcessCleanupLog(t, logDir, "stale.err") + currentStdout := createPersistentProcessCleanupLog(t, logDir, "current.out") + currentStderr := createPersistentProcessCleanupLog(t, logDir, "current.err") + + staleRecord := statestore.PersistentProcessRecord{ + ResourceKey: "default/reloaded", + LifecycleKey: "stale-lifecycle", + PID: leaseOwner.Pid, + IdentityTime: time.Unix(1, 0).UTC(), + RunID: "stale", + StdOutFile: staleStdout, + StdErrFile: staleStderr, + } + currentRecord := statestore.PersistentProcessRecord{ + ResourceKey: staleRecord.ResourceKey, + LifecycleKey: "current-lifecycle", + PID: leaseOwner.Pid, + IdentityTime: leaseOwner.IdentityTime, + RunID: "current", + StdOutFile: currentStdout, + StdErrFile: currentStderr, + } + require.NoError(t, stateStore.UpsertPersistentProcess(ctx, currentRecord)) + + require.NoError(t, cleanupInvalidPersistentExecutableRecord(ctx, stateStore, leaseOwner, staleRecord, logr.Discard())) + + actual, getErr := stateStore.GetPersistentProcess(ctx, currentRecord.ResourceKey) + require.NoError(t, getErr) + require.Equal(t, currentRecord.RunID, actual.RunID) + require.FileExists(t, currentStdout) + require.FileExists(t, currentStderr) + require.FileExists(t, staleStdout) + require.FileExists(t, staleStderr) +} + +func TestListPersistentProcessesReturnsRecordsInResourceKeyOrder(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + stateStore := openPersistentProcessCleanupTestStore(t, ctx) + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + require.NoError(t, leaseOwnerErr) + + for _, resourceKey := range []string{"default/z", "default/a"} { + require.NoError(t, stateStore.UpsertPersistentProcess(ctx, statestore.PersistentProcessRecord{ + ResourceKey: resourceKey, + LifecycleKey: resourceKey + "-lifecycle", + PID: leaseOwner.Pid, + IdentityTime: leaseOwner.IdentityTime, + RunID: resourceKey + "-run", + })) + } + + records, listErr := stateStore.ListPersistentProcesses(ctx) + require.NoError(t, listErr) + require.Len(t, records, 2) + require.Equal(t, "default/a", records[0].ResourceKey) + require.Equal(t, "default/z", records[1].ResourceKey) +} + +func openPersistentProcessCleanupTestStore(t *testing.T, ctx context.Context) *statestore.Store { + t.Helper() + + stateStorePath := filepath.Join(t.TempDir(), "state-store", "state.sqlite3") + stateStore, openErr := statestore.Open(ctx, statestore.Options{ + Path: stateStorePath, + BusyTimeout: 500 * time.Millisecond, + }) + require.NoError(t, openErr) + t.Cleanup(func() { + require.NoError(t, stateStore.Close()) + }) + return stateStore +} + +func createPersistentProcessCleanupLog(t *testing.T, dir string, name string) string { + t.Helper() + + path := filepath.Join(dir, name) + file, openErr := usvc_io.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) + require.NoError(t, openErr) + require.NoError(t, file.Close()) + t.Cleanup(func() { + removeErr := os.Remove(path) + if removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + require.NoError(t, removeErr) + } + }) + return path +} diff --git a/internal/dcpctrl/commands/run_controllers.go b/internal/dcpctrl/commands/run_controllers.go index 24f80ace..2ccafa42 100644 --- a/internal/dcpctrl/commands/run_controllers.go +++ b/internal/dcpctrl/commands/run_controllers.go @@ -29,6 +29,7 @@ import ( "github.com/microsoft/dcp/internal/notifications" "github.com/microsoft/dcp/internal/perftrace" "github.com/microsoft/dcp/internal/proxy" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/pkg/kubeconfig" "github.com/microsoft/dcp/pkg/logger" "github.com/microsoft/dcp/pkg/process" @@ -116,6 +117,24 @@ func runControllers(log logr.Logger) func(cmd *cobra.Command, _ []string) error return fmt.Errorf("cannot set up connection to the API server without kubeconfig file: %w", err) } + stateStore, stateStoreErr := statestore.Open(ctrlCtx, statestore.Options{}) + if stateStoreErr != nil { + return fmt.Errorf("failed to initialize state store: %w", stateStoreErr) + } + defer func() { + if stateStoreCloseErr := stateStore.Close(); stateStoreCloseErr != nil { + log.Error(stateStoreCloseErr, "Failed to close state store") + } + }() + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + if leaseOwnerErr != nil { + return fmt.Errorf("failed to initialize state store lease owner identity: %w", leaseOwnerErr) + } + if cleanupErr := stateStore.DeleteInactiveResourceLeases(ctrlCtx); cleanupErr != nil { + log.Error(cleanupErr, "Failed to clean up inactive state store resource leases") + } + startInvalidPersistentExecutableRecordCleanup(ctrlCtx, stateStore, leaseOwner, log) + trySetupNotificationHandler(ctrlCtx, log) mgr, err := getManager(ctrlCtx, log.V(1)) @@ -175,13 +194,17 @@ func runControllers(log logr.Logger) func(cmd *cobra.Command, _ []string) error return err } - exCtrl := controllers.NewExecutableReconciler( + exCtrl := controllers.NewExecutableReconcilerWithConfig( ctrlCtx, mgr.GetClient(), mgr.GetAPIReader(), log.WithName("ExecutableReconciler"), exeRunners, hpSet, + controllers.ExecutableReconcilerConfig{ + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + }, ) if err = exCtrl.SetupWithManager(mgr, defaultControllerName); err != nil { log.Error(err, "Unable to set up Executable controller") @@ -208,6 +231,9 @@ func runControllers(log logr.Logger) func(cmd *cobra.Command, _ []string) error hpSet, controllers.ContainerReconcilerConfig{ MaxParallelContainerStarts: controllers.DefaultMaxParallelContainerStarts, + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + ProcessExecutor: processExecutor, }, ) if err = containerCtrl.SetupWithManager(mgr, defaultControllerName); err != nil { @@ -239,13 +265,17 @@ func runControllers(log logr.Logger) func(cmd *cobra.Command, _ []string) error return err } - networkCtrl := controllers.NewNetworkReconciler( + networkCtrl := controllers.NewNetworkReconcilerWithConfig( ctrlCtx, mgr.GetClient(), mgr.GetAPIReader(), log.WithName("NetworkReconciler"), containerOrchestrator, harvester, + controllers.NetworkReconcilerConfig{ + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + }, ) if err = networkCtrl.SetupWithManager(mgr, defaultControllerName); err != nil { log.Error(err, "Unable to setup a ContainerNetwork controller") diff --git a/internal/dcpproc/dcpproc_api.go b/internal/dcpproc/dcpproc_api.go index 25979598..7213abdb 100644 --- a/internal/dcpproc/dcpproc_api.go +++ b/internal/dcpproc/dcpproc_api.go @@ -15,6 +15,7 @@ import ( "time" "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/microsoft/dcp/internal/dcppaths" internal_testutil "github.com/microsoft/dcp/internal/testutil" @@ -27,6 +28,25 @@ const ( DCP_DISABLE_MONITOR_PROCESS = "DCP_DISABLE_MONITOR_PROCESS" ) +func MonitorTargetFromFields(monitorPID *int64, monitorTimestamp metav1.MicroTime) (process.ProcessTreeItem, bool, error) { + if monitorPID == nil { + return process.ProcessTreeItem{}, false, nil + } + if monitorTimestamp.IsZero() { + return process.ProcessTreeItem{}, false, fmt.Errorf("monitor timestamp must be set when monitor PID is set") + } + + pid, pidErr := process.Int64_ToPidT(*monitorPID) + if pidErr != nil { + return process.ProcessTreeItem{}, false, fmt.Errorf("invalid monitor PID %d: %w", *monitorPID, pidErr) + } + + return process.ProcessTreeItem{ + Pid: pid, + IdentityTime: monitorTimestamp.Time, + }, true, nil +} + // Starts the process monitor for the given child process, using current process as the "monitored", or watched, process. // The caller should ensure that the current process is the correct process to monitor. // Errors are logged, but no error is returned if the process monitor fails to start-- @@ -38,6 +58,18 @@ func RunProcessWatcher( childPid process.Pid_t, childStartTime time.Time, log logr.Logger, +) { + monitorPid := process.Uint32_ToPidT(uint32(os.Getpid())) + monitorIdentityTime := process.ProcessIdentityTime(monitorPid) + RunProcessWatcherForMonitor(pe, process.ProcessTreeItem{Pid: monitorPid, IdentityTime: monitorIdentityTime}, childPid, childStartTime, log) +} + +func RunProcessWatcherForMonitor( + pe process.Executor, + monitor process.ProcessTreeItem, + childPid process.Pid_t, + childStartTime time.Time, + log logr.Logger, ) { if _, found := os.LookupEnv(DCP_DISABLE_MONITOR_PROCESS); found { return @@ -52,7 +84,7 @@ func RunProcessWatcher( if !childStartTime.IsZero() { cmdArgs = append(cmdArgs, "--child-identity-time", childStartTime.Format(osutil.RFC3339MiliTimestampFormat)) } - cmdArgs = append(cmdArgs, getMonitorCmdArgs()...) + cmdArgs = append(cmdArgs, getMonitorCmdArgs(monitor)...) startErr := startDcpProc(pe, cmdArgs) if startErr != nil { @@ -68,6 +100,17 @@ func RunContainerWatcher( pe process.Executor, containerID string, log logr.Logger, +) { + monitorPid := process.Uint32_ToPidT(uint32(os.Getpid())) + monitorIdentityTime := process.ProcessIdentityTime(monitorPid) + RunContainerWatcherForMonitor(pe, process.ProcessTreeItem{Pid: monitorPid, IdentityTime: monitorIdentityTime}, containerID, log) +} + +func RunContainerWatcherForMonitor( + pe process.Executor, + monitor process.ProcessTreeItem, + containerID string, + log logr.Logger, ) { if _, found := os.LookupEnv(DCP_DISABLE_MONITOR_PROCESS); found { return @@ -79,7 +122,7 @@ func RunContainerWatcher( "monitor-container", "--containerID", containerID, } - cmdArgs = append(cmdArgs, getMonitorCmdArgs()...) + cmdArgs = append(cmdArgs, getMonitorCmdArgs(monitor)...) startErr := startDcpProc(pe, cmdArgs) if startErr != nil { @@ -127,17 +170,13 @@ func StopProcessTree( return nil } -func getMonitorCmdArgs() []string { - monitorPid := os.Getpid() - +func getMonitorCmdArgs(monitor process.ProcessTreeItem) []string { // Add monitor PID to the command args - cmdArgs := []string{"--monitor", strconv.Itoa(monitorPid)} + cmdArgs := []string{"--monitor", strconv.FormatInt(int64(monitor.Pid), 10)} // Add monitor start time if available - rootPid := process.Uint32_ToPidT(uint32(monitorPid)) - identityTime := process.ProcessIdentityTime(rootPid) - if !identityTime.IsZero() { - cmdArgs = append(cmdArgs, "--monitor-identity-time", identityTime.Format(osutil.RFC3339MiliTimestampFormat)) + if !monitor.IdentityTime.IsZero() { + cmdArgs = append(cmdArgs, "--monitor-identity-time", monitor.IdentityTime.Format(osutil.RFC3339MiliTimestampFormat)) } return cmdArgs diff --git a/internal/dcpproc/dcpproc_api_test.go b/internal/dcpproc/dcpproc_api_test.go index b15d3f28..ab36aac1 100644 --- a/internal/dcpproc/dcpproc_api_test.go +++ b/internal/dcpproc/dcpproc_api_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/microsoft/dcp/internal/dcppaths" internal_testutil "github.com/microsoft/dcp/internal/testutil" @@ -23,6 +24,15 @@ import ( "github.com/microsoft/dcp/pkg/testutil" ) +func TestMonitorTargetFromFieldsRequiresTimestamp(t *testing.T) { + monitorPID := int64(12345) + + _, found, monitorErr := MonitorTargetFromFields(&monitorPID, metav1.MicroTime{}) + + require.False(t, found) + require.ErrorContains(t, monitorErr, "monitor timestamp must be set when monitor PID is set") +} + func TestRunProcessWatcher(t *testing.T) { log := testutil.NewLogForTesting(t.Name()) ctx, cancel := testutil.GetTestContext(t, 20*time.Second) @@ -52,6 +62,36 @@ func TestRunProcessWatcher(t *testing.T) { require.Equal(t, dcpProc.Cmd.Args[7], strconv.FormatInt(int64(os.Getpid()), 10), "Should include current process PID as monitored PID") } +func TestRunProcessWatcherForMonitor(t *testing.T) { + log := testutil.NewLogForTesting(t.Name()) + ctx, cancel := testutil.GetTestContext(t, 20*time.Second) + defer cancel() + pe := internal_testutil.NewTestProcessExecutor(ctx) + dcppaths.EnableTestPathProbing() + + testPid := process.Pid_t(28869) + testStartTime := time.Now() + monitor := process.ProcessTreeItem{ + Pid: 12345, + IdentityTime: testStartTime.Add(-time.Minute), + } + + RunProcessWatcherForMonitor(pe, monitor, testPid, testStartTime, log) + + dcpProc, dcpProcErr := findRunningDcp(pe) + require.NoError(t, dcpProcErr) + + require.Equal(t, "monitor-process", dcpProc.Cmd.Args[1], "Should use 'monitor-process' subcommand") + require.Equal(t, "--child", dcpProc.Cmd.Args[2], "Should include --child flag") + require.Equal(t, strconv.FormatInt(int64(testPid), 10), dcpProc.Cmd.Args[3], "Should include child PID") + require.Equal(t, "--child-identity-time", dcpProc.Cmd.Args[4], "Should include --child-identity-time flag") + require.Equal(t, testStartTime.Format(osutil.RFC3339MiliTimestampFormat), dcpProc.Cmd.Args[5], "Should include formatted child start time") + require.Equal(t, "--monitor", dcpProc.Cmd.Args[6], "Should include --monitor flag") + require.Equal(t, strconv.FormatInt(int64(monitor.Pid), 10), dcpProc.Cmd.Args[7], "Should include explicit monitored PID") + require.Equal(t, "--monitor-identity-time", dcpProc.Cmd.Args[8], "Should include --monitor-identity-time flag") + require.Equal(t, monitor.IdentityTime.Format(osutil.RFC3339MiliTimestampFormat), dcpProc.Cmd.Args[9], "Should include explicit monitor identity time") +} + func TestRunContainerWatcher(t *testing.T) { log := testutil.NewLogForTesting(t.Name()) ctx, cancel := testutil.GetTestContext(t, 20*time.Second) @@ -78,6 +118,33 @@ func TestRunContainerWatcher(t *testing.T) { require.Equal(t, dcpProc.Cmd.Args[5], strconv.FormatInt(int64(os.Getpid()), 10), "Should include current process PID as monitored PID") } +func TestRunContainerWatcherForMonitor(t *testing.T) { + log := testutil.NewLogForTesting(t.Name()) + ctx, cancel := testutil.GetTestContext(t, 20*time.Second) + defer cancel() + pe := internal_testutil.NewTestProcessExecutor(ctx) + dcppaths.EnableTestPathProbing() + + testContainerID := "test-container-123" + monitor := process.ProcessTreeItem{ + Pid: 12345, + IdentityTime: time.Now().Add(-time.Minute), + } + + RunContainerWatcherForMonitor(pe, monitor, testContainerID, log) + + dcpProc, dcpProcErr := findRunningDcp(pe) + require.NoError(t, dcpProcErr) + + require.Equal(t, "monitor-container", dcpProc.Cmd.Args[1], "Should use 'monitor-container' subcommand") + require.Equal(t, "--containerID", dcpProc.Cmd.Args[2], "Should include --containerID flag") + require.Equal(t, testContainerID, dcpProc.Cmd.Args[3], "Should include container ID") + require.Equal(t, "--monitor", dcpProc.Cmd.Args[4], "Should include --monitor flag") + require.Equal(t, strconv.FormatInt(int64(monitor.Pid), 10), dcpProc.Cmd.Args[5], "Should include explicit monitored PID") + require.Equal(t, "--monitor-identity-time", dcpProc.Cmd.Args[6], "Should include --monitor-identity-time flag") + require.Equal(t, monitor.IdentityTime.Format(osutil.RFC3339MiliTimestampFormat), dcpProc.Cmd.Args[7], "Should include explicit monitor identity time") +} + func TestStopProcessTree(t *testing.T) { log := testutil.NewLogForTesting(t.Name()) ctx, cancel := testutil.GetTestContext(t, 20*time.Second) diff --git a/internal/exerunners/ide_executable_runner.go b/internal/exerunners/ide_executable_runner.go index 9f6999e9..8d53c82f 100644 --- a/internal/exerunners/ide_executable_runner.go +++ b/internal/exerunners/ide_executable_runner.go @@ -335,6 +335,16 @@ func (r *IdeExecutableRunner) StopRun(ctx context.Context, runID controllers.Run return fmt.Errorf(runSessionCouldNotBeStopped+"%s %s", resp.Status, parseResponseBody(respBody)) } +func (r *IdeExecutableRunner) ReleaseRun(_ context.Context, runID controllers.RunID, log logr.Logger) error { + rd, found := r.activeRuns.LoadAndDelete(runID) + if !found { + log.V(1).Info("Release of an IDE run session requested, but the run was already released", "RunID", runID) + return nil + } + rd.CloseOutputWriters() + return nil +} + func (r *IdeExecutableRunner) prepareRunRequest(exe *apiv1.Executable) (*http.Request, context.CancelFunc, error) { var isrBody []byte var bodyErr error diff --git a/internal/exerunners/process_executable_runner.go b/internal/exerunners/process_executable_runner.go index d37982c5..6ba9fea9 100644 --- a/internal/exerunners/process_executable_runner.go +++ b/internal/exerunners/process_executable_runner.go @@ -9,10 +9,12 @@ import ( "context" "errors" "fmt" - "io" "os" "os/exec" + "path/filepath" "strconv" + "strings" + "sync" "time" "github.com/go-logr/logr" @@ -30,22 +32,51 @@ import ( "github.com/microsoft/dcp/pkg/syncmap" ) +const ( + DCP_PERSISTENT_EXECUTABLE_OUTPUT_DIR = "DCP_PERSISTENT_EXECUTABLE_OUTPUT_DIR" + + persistentExecutableOutputDirName = "dcp-peo" +) + +var persistentExecutableOutputDir = func() (string, error) { + isAdmin, adminErr := osutil.IsAdmin() + if adminErr != nil { + return "", fmt.Errorf("could not determine current elevation: %w", adminErr) + } + baseDir := persistentExecutableOutputBaseDir() + if isAdmin { + return baseDir + "-admin", nil + } + return baseDir, nil +} + +func persistentExecutableOutputBaseDir() string { + if outputDir, found := os.LookupEnv(DCP_PERSISTENT_EXECUTABLE_OUTPUT_DIR); found && strings.TrimSpace(outputDir) != "" { + return strings.TrimSpace(outputDir) + } + return filepath.Join(os.TempDir(), persistentExecutableOutputDirName) +} + type processRunState struct { - identityTime time.Time - stdOutFile *os.File - stdErrFile *os.File - cmdInfo string // Command line used to start the process, for logging purposes + pid process.Pid_t + identityTime time.Time + stdOutFile *os.File + stdErrFile *os.File + cmdInfo string // Command line used to start the process, for logging purposes + adopted bool + cancelWatch func() + runChangeHandler controllers.RunChangeHandler } type ProcessExecutableRunner struct { pe process.Executor - runningProcesses *syncmap.Map[controllers.RunID, *processRunState] + runningProcesses *syncmap.ComparableValueMap[controllers.RunID, *processRunState] } func NewProcessExecutableRunner(pe process.Executor) *ProcessExecutableRunner { return &ProcessExecutableRunner{ pe: pe, - runningProcesses: &syncmap.Map[controllers.RunID, *processRunState]{}, + runningProcesses: &syncmap.ComparableValueMap[controllers.RunID, *processRunState]{}, } } @@ -68,19 +99,19 @@ func (r *ProcessExecutableRunner) StartRun( startLog.V(1).Info("Process details", "Env", cmd.Env, "Cwd", cmd.Dir) result := controllers.NewExecutableStartResult() - stdOutFile, stdOutFileErr := usvc_io.OpenTempFile(fmt.Sprintf("%s_out_%s", exe.Name, exe.UID), os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) + stdOutFile, stdOutFileErr := openExecutableOutputFile(exe, "out") if stdOutFileErr != nil { startLog.Error(stdOutFileErr, "Failed to create temporary file for capturing process standard output data") } else { - cmd.Stdout = usvc_io.NewTimestampWriter(stdOutFile) + cmd.Stdout = executableOutputWriter(exe, stdOutFile) result.StdOutFile = stdOutFile.Name() } - stdErrFile, stdErrFileErr := usvc_io.OpenTempFile(fmt.Sprintf("%s_err_%s", exe.Name, exe.UID), os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) + stdErrFile, stdErrFileErr := openExecutableOutputFile(exe, "err") if stdErrFileErr != nil { startLog.Error(stdErrFileErr, "Failed to create temporary file for capturing process standard error data") } else { - cmd.Stderr = usvc_io.NewTimestampWriter(stdErrFile) + cmd.Stderr = executableOutputWriter(exe, stdErrFile) result.StdErrFile = stdErrFile.Name() } @@ -91,13 +122,7 @@ func (r *ProcessExecutableRunner) StartRun( runID := pidToRunID(pid) if runState, found := r.runningProcesses.LoadAndDelete(runID); found { - for _, f := range []io.Closer{runState.stdOutFile, runState.stdErrFile} { - if f != nil { - if closeErr := f.Close(); closeErr != nil && !errors.Is(closeErr, os.ErrClosed) { - err = errors.Join(closeErr, err) - } - } - } + err = errors.Join(closeProcessRunFiles(runState), err) } if runChangeHandler != nil { @@ -105,8 +130,14 @@ func (r *ProcessExecutableRunner) StartRun( } }) - // We want to ensure that the service process tree is killed when DCP is stopped so that ports are released etc. - pid, processIdentityTime, startWaitForProcessExit, startErr := r.pe.StartProcess(ctx, cmd, processExitHandler, process.CreationFlagEnsureKillOnDispose) + var creationFlags process.ProcessCreationFlag = process.CreationFlagEnsureKillOnDispose + processCtx := ctx + if exe.Spec.Persistent { + creationFlags = process.CreationFlagsNone + processCtx = context.WithoutCancel(ctx) + } + + pid, processIdentityTime, startWaitForProcessExit, startErr := r.pe.StartProcess(processCtx, cmd, processExitHandler, creationFlags) if startErr != nil { startLog.Error(startErr, "Failed to start a process") result.CompletionTimestamp = metav1.NowMicro() @@ -126,20 +157,31 @@ func (r *ProcessExecutableRunner) StartRun( runChangeHandler.OnStartupCompleted(exe.NamespacedName(), result) return result } else { - // Use original log here, the watcher is a different process. - dcpproc.RunProcessWatcher(r.pe, pid, processIdentityTime, log) + if !exe.Spec.Persistent { + // Use original log here, the watcher is a different process. + dcpproc.RunProcessWatcher(r.pe, pid, processIdentityTime, log) + } else if monitor, found, monitorErr := dcpproc.MonitorTargetFromFields(exe.Spec.MonitorPID, exe.Spec.MonitorTimestamp); monitorErr != nil { + log.Error(monitorErr, "Could not start persistent Executable lifecycle monitor") + } else if found { + // Use original log here, the watcher is a different process. + dcpproc.RunProcessWatcherForMonitor(r.pe, monitor, pid, processIdentityTime, log) + } r.runningProcesses.Store(pidToRunID(pid), &processRunState{ - identityTime: processIdentityTime, - stdOutFile: stdOutFile, - stdErrFile: stdErrFile, - cmdInfo: cmd.String(), + pid: pid, + identityTime: processIdentityTime, + stdOutFile: stdOutFile, + stdErrFile: stdErrFile, + cmdInfo: cmd.String(), + runChangeHandler: runChangeHandler, }) + displayStartTime := process.StartTimeForProcess(pid) result.RunID = pidToRunID(pid) pointers.SetValue(&result.Pid, int64(pid)) result.ExeState = apiv1.ExecutableStateRunning - result.CompletionTimestamp = metav1.NewMicroTime(process.StartTimeForProcess(pid)) + result.CompletionTimestamp = metav1.NewMicroTime(displayStartTime) + result.ProcessIdentityTime = processIdentityTime result.StartWaitForRunCompletion = startWaitForProcessExit runChangeHandler.OnStartupCompleted(exe.NamespacedName(), result) @@ -148,12 +190,98 @@ func (r *ProcessExecutableRunner) StartRun( } } +func (r *ProcessExecutableRunner) AdoptRun( + _ context.Context, + run controllers.ExecutableRunAdoptionInfo, + runChangeHandler controllers.RunChangeHandler, + log logr.Logger, +) error { + if run.RunID == controllers.UnknownRunID { + return fmt.Errorf("cannot adopt a process run without a run ID") + } + if run.Pid == process.UnknownPID { + return fmt.Errorf("cannot adopt process run %s without a valid PID", run.RunID) + } + if run.ProcessIdentityTime.IsZero() { + return fmt.Errorf("cannot adopt process run %s without process identity time", run.RunID) + } + + if _, findErr := process.FindProcess(run.Pid, run.ProcessIdentityTime); findErr != nil { + return fmt.Errorf("cannot adopt process run %s: %w", run.RunID, findErr) + } + stopWatching := make(chan struct{}) + var stopWatchingOnce sync.Once + cancelWatch := func() { + stopWatchingOnce.Do(func() { + close(stopWatching) + }) + } + + r.runningProcesses.Store(run.RunID, &processRunState{ + pid: run.Pid, + identityTime: run.ProcessIdentityTime, + cmdInfo: run.CommandInfo, + adopted: true, + cancelWatch: cancelWatch, + runChangeHandler: runChangeHandler, + }) + + go r.watchAdoptedProcess(run.RunID, run.Pid, run.ProcessIdentityTime, stopWatching, log) + + return nil +} + +func (r *ProcessExecutableRunner) watchAdoptedProcess( + runID controllers.RunID, + pid process.Pid_t, + identityTime time.Time, + stopWatching <-chan struct{}, + log logr.Logger, +) { + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + + for { + select { + case <-stopWatching: + return + + case <-timer.C: + if _, findErr := process.FindProcess(pid, identityTime); findErr != nil { + runState, found := r.runningProcesses.Load(runID) + if !found { + return + } + if runState.pid != pid || !runState.identityTime.Equal(identityTime) { + return + } + if !r.runningProcesses.CompareAndDelete(runID, runState) { + return + } + + waitErr := closeProcessRunFiles(runState) + if runState.runChangeHandler != nil { + runState.runChangeHandler.OnRunCompleted(runID, apiv1.UnknownExitCode, waitErr) + } + if waitErr != nil { + log.Error(waitErr, "Error watching adopted process", "RunID", runID, "PID", runState.pid) + } + return + } + timer.Reset(2 * time.Second) + } + } +} + func (r *ProcessExecutableRunner) StopRun(ctx context.Context, runID controllers.RunID, log logr.Logger) error { runState, found := r.runningProcesses.LoadAndDelete(runID) if !found { log.V(1).Info("Stop of a process run requested, but the run was already stopped", "RunID", runID) return nil } + if runState.cancelWatch != nil { + runState.cancelWatch() + } stopLog := log.WithValues("RunID", runID, "Command", runState.cmdInfo) stopLog.V(1).Info("Stopping run...") @@ -166,15 +294,7 @@ func (r *ProcessExecutableRunner) StopRun(ctx context.Context, runID controllers errCh := make(chan error, 1) go func() { - if osutil.IsWindows() { - // See StartRun() for why we need to use separate console for the app process on Windows. - // This means we cannot send Ctrl-C to that process directly and need to use dcpproc StopProcessTree facility instead. - stopCtx, stopCtxCancel := context.WithTimeout(ctx, ProcessStopTimeout) - defer stopCtxCancel() - errCh <- dcpproc.StopProcessTree(stopCtx, r.pe, runIdToPID(runID), runState.identityTime, stopLog) - } else { - errCh <- r.pe.StopProcess(runIdToPID(runID), runState.identityTime) - } + errCh <- process.StopViaConsole(stopLog, r.pe, runState.pid, runState.identityTime) }() var stopErr error = nil @@ -186,21 +306,73 @@ func (r *ProcessExecutableRunner) StopRun(ctx context.Context, runID controllers } if found { - for _, f := range []io.Closer{runState.stdOutFile, runState.stdErrFile} { - if f != nil { - if err := f.Close(); err != nil && !errors.Is(err, os.ErrClosed) { - stopErr = errors.Join(stopErr, err) - } - } - } + stopErr = errors.Join(stopErr, closeProcessRunFiles(runState)) } if stopErr != nil { stopLog.Error(stopErr, "Failed to stop run") + } else if runState.adopted && runState.runChangeHandler != nil { + runState.runChangeHandler.OnRunCompleted(runID, apiv1.UnknownExitCode, nil) } return stopErr } +func (r *ProcessExecutableRunner) ReleaseRun(_ context.Context, runID controllers.RunID, log logr.Logger) error { + runState, found := r.runningProcesses.LoadAndDelete(runID) + if !found { + log.V(1).Info("Release of a process run requested, but the run was already released", "RunID", runID) + return nil + } + if runState.cancelWatch != nil { + runState.cancelWatch() + } + + closeErr := closeProcessRunFiles(runState) + if closeErr != nil { + log.Error(closeErr, "Could not close process run files while releasing run", "RunID", runID) + } + return closeErr +} + +func closeProcessRunFiles(runState *processRunState) error { + var closeErr error + for _, file := range []*os.File{runState.stdOutFile, runState.stdErrFile} { + if file != nil { + fileErr := file.Close() + if fileErr != nil && !errors.Is(fileErr, os.ErrClosed) { + closeErr = errors.Join(closeErr, fileErr) + } + } + } + return closeErr +} + +func openExecutableOutputFile(exe *apiv1.Executable, stream string) (*os.File, error) { + // Persistent output files are keyed by resource UID to keep paths unique across persistent Executable instances. + // Kubernetes UIDs are effectively unique, so collisions between different Executable objects are not expected. + fileName := fmt.Sprintf("%s_%s", exe.UID, stream) + if !exe.Spec.Persistent { + return usvc_io.OpenTempFile(fileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) + } + + dir, dirErr := persistentExecutableOutputDir() + if dirErr != nil { + return nil, fmt.Errorf("could not determine persistent Executable output directory: %w", dirErr) + } + if ensureDirErr := usvc_io.EnsureRestrictedDirectory(dir, osutil.PermissionOnlyOwnerReadWriteTraverse); ensureDirErr != nil { + return nil, fmt.Errorf("could not prepare persistent Executable output directory: %w", ensureDirErr) + } + + return usvc_io.OpenFile(filepath.Join(dir, fileName), os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) +} + +func executableOutputWriter(exe *apiv1.Executable, file *os.File) usvc_io.WriteSyncerCloser { + if exe.Spec.Persistent { + return file + } + return usvc_io.NewTimestampWriter(file) +} + func makeCommand(exe *apiv1.Executable) *exec.Cmd { cmd := exec.Command(exe.Spec.ExecutablePath) cmd.Args = append([]string{exe.Spec.ExecutablePath}, exe.Status.EffectiveArgs...) @@ -216,16 +388,5 @@ func pidToRunID(pid process.Pid_t) controllers.RunID { return controllers.RunID(strconv.FormatInt(int64(pid), 10)) } -func runIdToPID(runID controllers.RunID) process.Pid_t { - pid64, err := strconv.ParseInt(string(runID), 10, 64) - if err != nil { - return process.UnknownPID - } - pid, err := process.Int64_ToPidT(pid64) - if err != nil { - return process.UnknownPID - } - return pid -} - var _ controllers.ExecutableRunner = (*ProcessExecutableRunner)(nil) +var _ controllers.PersistentExecutableRunner = (*ProcessExecutableRunner)(nil) diff --git a/internal/exerunners/process_executable_runner_test.go b/internal/exerunners/process_executable_runner_test.go new file mode 100644 index 00000000..a42165d0 --- /dev/null +++ b/internal/exerunners/process_executable_runner_test.go @@ -0,0 +1,433 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package exerunners + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/controllers" + "github.com/microsoft/dcp/internal/dcppaths" + "github.com/microsoft/dcp/internal/testutil" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/process" +) + +func TestProcessExecutableRunnerStartsLifecycleMonitor(t *testing.T) { + monitorPID := int64(12345) + monitorTimestamp := metav1.NewMicroTime(time.Now().Add(-time.Minute)) + testCases := []struct { + name string + persistent bool + monitorPID *int64 + monitorTimestamp metav1.MicroTime + expectedMonitorStarts int + }{ + { + name: "non-persistent executable starts monitor", + expectedMonitorStarts: 1, + }, + { + name: "persistent executable skips monitor", + persistent: true, + }, + { + name: "persistent executable with monitor starts monitor", + persistent: true, + monitorPID: &monitorPID, + monitorTimestamp: monitorTimestamp, + expectedMonitorStarts: 1, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + dcppaths.EnableTestPathProbing() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + processExecutor := testutil.NewTestProcessExecutor(ctx) + runner := NewProcessExecutableRunner(processExecutor) + persistentOutputDir := "" + if testCase.persistent { + persistentOutputDir = overridePersistentExecutableOutputDir(t) + } + exe := &apiv1.Executable{ + ObjectMeta: metav1.ObjectMeta{ + Name: "api", + Namespace: "default", + UID: "api-uid", + }, + Spec: apiv1.ExecutableSpec{ + ExecutablePath: "/test/app", + Persistent: testCase.persistent, + MonitorPID: testCase.monitorPID, + MonitorTimestamp: testCase.monitorTimestamp, + }, + } + + result := runner.StartRun(ctx, exe, &testRunChangeHandler{}, logr.Discard()) + + require.Equal(t, apiv1.ExecutableStateRunning, result.ExeState) + t.Cleanup(func() { + require.NoError(t, runner.ReleaseRun(context.Background(), result.RunID, logr.Discard())) + removeFileIfExists(t, result.StdOutFile) + removeFileIfExists(t, result.StdErrFile) + }) + if testCase.persistent { + require.Equal(t, persistentOutputDir, filepath.Dir(result.StdOutFile)) + require.Equal(t, persistentOutputDir, filepath.Dir(result.StdErrFile)) + } + require.Len(t, processExecutor.FindAll([]string{"/test/app"}, "", nil), 1) + monitorProcesses := processExecutor.FindAll([]string{"dcp", "monitor-process"}, "", nil) + require.Len(t, monitorProcesses, testCase.expectedMonitorStarts) + if testCase.monitorPID != nil { + require.Contains(t, monitorProcesses[0].Cmd.Args, "--monitor") + require.Contains(t, monitorProcesses[0].Cmd.Args, strconv.FormatInt(*testCase.monitorPID, 10)) + require.Contains(t, monitorProcesses[0].Cmd.Args, "--monitor-identity-time") + require.Contains(t, monitorProcesses[0].Cmd.Args, testCase.monitorTimestamp.Time.Format(osutil.RFC3339MiliTimestampFormat)) + } + }) + } +} + +func TestPersistentExecutableOutputFileUsesPersistentOutputDir(t *testing.T) { + persistentOutputDir := overridePersistentExecutableOutputDir(t) + exe := &apiv1.Executable{ + ObjectMeta: metav1.ObjectMeta{ + Name: "api/name", + Namespace: "default", + UID: "api-uid", + }, + Spec: apiv1.ExecutableSpec{ + LifecycleKey: "life/key", + Persistent: true, + }, + } + + file, fileErr := openExecutableOutputFile(exe, "out") + require.NoError(t, fileErr) + t.Cleanup(func() { + require.NoError(t, file.Close()) + removeFileIfExists(t, file.Name()) + }) + + require.Equal(t, persistentOutputDir, filepath.Dir(file.Name())) + require.Equal(t, "api-uid_out", filepath.Base(file.Name())) +} + +func TestPersistentExecutableOutputBaseDirCanBeConfigured(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "custom-peo") + t.Setenv(DCP_PERSISTENT_EXECUTABLE_OUTPUT_DIR, outputDir) + + require.Equal(t, outputDir, persistentExecutableOutputBaseDir()) +} + +func TestPersistentExecutableOutputBaseDirDefaultsForEmptyEnvVar(t *testing.T) { + t.Setenv(DCP_PERSISTENT_EXECUTABLE_OUTPUT_DIR, " ") + + require.Equal(t, filepath.Join(os.TempDir(), persistentExecutableOutputDirName), persistentExecutableOutputBaseDir()) +} + +func TestProcessExecutableRunnerSkipsTimestampsForPersistentOutput(t *testing.T) { + testCases := []struct { + name string + persistent bool + }{ + { + name: "non-persistent output is timestamped", + }, + { + name: "persistent output is written directly", + persistent: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + dcppaths.EnableTestPathProbing() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if testCase.persistent { + overridePersistentExecutableOutputDir(t) + } + processExecutor := testutil.NewTestProcessExecutor(ctx) + runner := NewProcessExecutableRunner(processExecutor) + exe := &apiv1.Executable{ + ObjectMeta: metav1.ObjectMeta{ + Name: "api", + Namespace: "default", + UID: types.UID(fmt.Sprintf("api-uid-%d", time.Now().UnixNano())), + }, + Spec: apiv1.ExecutableSpec{ + ExecutablePath: "/test/app", + Persistent: testCase.persistent, + }, + } + + result := runner.StartRun(ctx, exe, &testRunChangeHandler{}, logr.Discard()) + require.Equal(t, apiv1.ExecutableStateRunning, result.ExeState) + t.Cleanup(func() { + require.NoError(t, runner.ReleaseRun(context.Background(), result.RunID, logr.Discard())) + removeFileIfExists(t, result.StdOutFile) + removeFileIfExists(t, result.StdErrFile) + }) + + executions := processExecutor.FindAll([]string{"/test/app"}, "", nil) + require.Len(t, executions, 1) + _, writeErr := executions[0].Cmd.Stdout.Write([]byte("hello\n")) + require.NoError(t, writeErr) + if syncer, ok := executions[0].Cmd.Stdout.(interface{ Sync() error }); ok { + require.NoError(t, syncer.Sync()) + } + + output, readErr := os.ReadFile(result.StdOutFile) + require.NoError(t, readErr) + if testCase.persistent { + require.Equal(t, "hello\n", string(output)) + } else { + require.True(t, strings.HasPrefix(string(output), "1 "), "expected timestamped output, got %q", string(output)) + require.Contains(t, string(output), "hello\n") + } + }) + } +} + +func TestAdoptedProcessStopUsesAdoptedPID(t *testing.T) { + t.Parallel() + + dcppaths.EnableTestPathProbing() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + processExecutor := &recordingProcessExecutor{} + runner := NewProcessExecutableRunner(processExecutor) + pid := process.Pid_t(42) + identityTime := time.Now().UTC() + originalRunID := pidToRunID(pid) + adoptedRunID := controllers.RunID(fmt.Sprintf("%s-adopted", originalRunID)) + runner.runningProcesses.Store(adoptedRunID, &processRunState{ + pid: pid, + identityTime: identityTime, + cmdInfo: "/test/app", + adopted: true, + }) + + require.NoError(t, runner.StopRun(ctx, adoptedRunID, logr.Discard())) + + require.Equal(t, pid, processExecutor.stoppedPID) + require.Equal(t, identityTime, processExecutor.stoppedIdentityTime) +} + +func TestAdoptedProcessReportsCompletionWhenProcessExits(t *testing.T) { + t.Parallel() + + delayToolDir, delayToolDirErr := testutil.GetTestToolDir("delay") + require.NoError(t, delayToolDirErr) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "./delay", "--delay=1s") + cmd.Dir = delayToolDir + require.NoError(t, cmd.Start()) + go func() { + _ = cmd.Wait() + }() + + pid := process.Uint32_ToPidT(uint32(cmd.Process.Pid)) + identityTime := process.ProcessIdentityTime(pid) + require.False(t, identityTime.IsZero()) + + runner := NewProcessExecutableRunner(&recordingProcessExecutor{}) + runID := pidToRunID(pid) + changeHandler := newRecordingRunChangeHandler() + + adoptErr := runner.AdoptRun(ctx, controllers.ExecutableRunAdoptionInfo{ + RunID: runID, + Pid: pid, + ProcessIdentityTime: identityTime, + CommandInfo: "./delay --delay=1s", + }, changeHandler, logr.Discard()) + require.NoError(t, adoptErr) + + select { + case completedRun := <-changeHandler.completedRuns: + require.Equal(t, runID, completedRun.runID) + require.Equal(t, apiv1.UnknownExitCode, completedRun.exitCode) + require.NoError(t, completedRun.err) + case <-ctx.Done(): + require.Fail(t, "timed out waiting for adopted process completion notification") + } + + _, found := runner.runningProcesses.Load(runID) + require.False(t, found) +} + +func TestAdoptedProcessWatcherDoesNotDeleteReusedRunID(t *testing.T) { + t.Parallel() + + runner := NewProcessExecutableRunner(&recordingProcessExecutor{}) + runID := controllers.RunID("42") + watchedPID := process.Pid_t(42) + watchedIdentityTime := time.Unix(1, 0).UTC() + reusedIdentityTime := watchedIdentityTime.Add(time.Minute) + changeHandler := newRecordingRunChangeHandler() + reusedRunState := &processRunState{ + pid: watchedPID, + identityTime: reusedIdentityTime, + runChangeHandler: changeHandler, + } + runner.runningProcesses.Store(runID, reusedRunState) + + runner.watchAdoptedProcess(runID, watchedPID, watchedIdentityTime, make(chan struct{}), logr.Discard()) + + storedRunState, found := runner.runningProcesses.Load(runID) + require.True(t, found) + require.Same(t, reusedRunState, storedRunState) + select { + case completedRun := <-changeHandler.completedRuns: + require.Failf(t, "unexpected completion notification", "received completion for run %s", completedRun.runID) + default: + } +} + +func TestReleaseRunClosesProcessRunFiles(t *testing.T) { + t.Parallel() + + stdOutFile, stdOutFileErr := usvc_io.OpenTempFile(fmt.Sprintf("stdout_%d", time.Now().UnixNano()), os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) + require.NoError(t, stdOutFileErr) + t.Cleanup(func() { + require.NoError(t, os.Remove(stdOutFile.Name())) + }) + stdErrFile, stdErrFileErr := usvc_io.OpenTempFile(fmt.Sprintf("stderr_%d", time.Now().UnixNano()), os.O_RDWR|os.O_CREATE|os.O_EXCL, osutil.PermissionOnlyOwnerReadWrite) + require.NoError(t, stdErrFileErr) + t.Cleanup(func() { + require.NoError(t, os.Remove(stdErrFile.Name())) + }) + + runner := NewProcessExecutableRunner(&recordingProcessExecutor{}) + runID := controllers.RunID("run-1") + runner.runningProcesses.Store(runID, &processRunState{ + stdOutFile: stdOutFile, + stdErrFile: stdErrFile, + }) + + require.NoError(t, runner.ReleaseRun(context.Background(), runID, logr.Discard())) + + require.ErrorIs(t, stdOutFile.Close(), os.ErrClosed) + require.ErrorIs(t, stdErrFile.Close(), os.ErrClosed) + _, found := runner.runningProcesses.Load(runID) + require.False(t, found) +} + +func overridePersistentExecutableOutputDir(t *testing.T) string { + t.Helper() + + outputDir := t.TempDir() + originalOutputDir := persistentExecutableOutputDir + persistentExecutableOutputDir = func() (string, error) { + return outputDir, nil + } + t.Cleanup(func() { + persistentExecutableOutputDir = originalOutputDir + }) + return outputDir +} + +func removeFileIfExists(t *testing.T, path string) { + t.Helper() + if path == "" { + return + } + removeErr := os.Remove(path) + if removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + require.NoError(t, removeErr) + } +} + +type recordingProcessExecutor struct { + stoppedPID process.Pid_t + stoppedIdentityTime time.Time +} + +func (e *recordingProcessExecutor) StartProcess(_ context.Context, _ *exec.Cmd, _ process.ProcessExitHandler, _ process.ProcessCreationFlag) (process.Pid_t, time.Time, func(), error) { + return process.UnknownPID, time.Time{}, nil, fmt.Errorf("process start is not supported by recordingProcessExecutor") +} + +func (e *recordingProcessExecutor) StopProcess(pid process.Pid_t, processStartTime time.Time, _ ...process.ProcessStopOption) error { + e.stoppedPID = pid + e.stoppedIdentityTime = processStartTime + return nil +} + +func (e *recordingProcessExecutor) StartAndForget(*exec.Cmd, process.ProcessCreationFlag) (process.Pid_t, time.Time, error) { + return process.UnknownPID, time.Time{}, fmt.Errorf("process start is not supported by recordingProcessExecutor") +} + +func (e *recordingProcessExecutor) Dispose() {} + +type testRunChangeHandler struct{} + +func (*testRunChangeHandler) OnMainProcessChanged(controllers.RunID, process.Pid_t) {} + +func (*testRunChangeHandler) OnRunCompleted(controllers.RunID, *int32, error) {} + +func (*testRunChangeHandler) OnStartupCompleted(types.NamespacedName, *controllers.ExecutableStartResult) { +} + +func (*testRunChangeHandler) OnRunMessage(controllers.RunID, controllers.RunMessageLevel, string) {} + +type completedRunNotification struct { + runID controllers.RunID + exitCode *int32 + err error +} + +type recordingRunChangeHandler struct { + completedRuns chan completedRunNotification +} + +func newRecordingRunChangeHandler() *recordingRunChangeHandler { + return &recordingRunChangeHandler{ + completedRuns: make(chan completedRunNotification, 1), + } +} + +func (*recordingRunChangeHandler) OnMainProcessChanged(controllers.RunID, process.Pid_t) {} + +func (h *recordingRunChangeHandler) OnRunCompleted(runID controllers.RunID, exitCode *int32, err error) { + h.completedRuns <- completedRunNotification{ + runID: runID, + exitCode: exitCode, + err: err, + } +} + +func (*recordingRunChangeHandler) OnStartupCompleted(types.NamespacedName, *controllers.ExecutableStartResult) { +} + +func (*recordingRunChangeHandler) OnRunMessage(controllers.RunID, controllers.RunMessageLevel, string) { +} diff --git a/internal/lockfile/lockfile_unix.go b/internal/lockfile/lockfile_unix.go index c4b45a6c..86aa8b1a 100644 --- a/internal/lockfile/lockfile_unix.go +++ b/internal/lockfile/lockfile_unix.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package lockfile import ( diff --git a/internal/lockfile/lockfile_windows.go b/internal/lockfile/lockfile_windows.go index d1291d86..81b418e4 100644 --- a/internal/lockfile/lockfile_windows.go +++ b/internal/lockfile/lockfile_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package lockfile import ( diff --git a/internal/networking/networking_darwin.go b/internal/networking/networking_darwin.go index 6fdef447..0bda099c 100644 --- a/internal/networking/networking_darwin.go +++ b/internal/networking/networking_darwin.go @@ -1,12 +1,10 @@ +//go:build darwin + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build darwin - -// Copyright (c) Microsoft Corporation. All rights reserved. - package networking import ( diff --git a/internal/networking/networking_linux.go b/internal/networking/networking_linux.go index 29e95df7..4ddbd4f6 100644 --- a/internal/networking/networking_linux.go +++ b/internal/networking/networking_linux.go @@ -1,12 +1,10 @@ +//go:build linux + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build linux - -// Copyright (c) Microsoft Corporation. All rights reserved. - package networking import ( diff --git a/internal/networking/networking_windows.go b/internal/networking/networking_windows.go index 85209fd1..be53a291 100644 --- a/internal/networking/networking_windows.go +++ b/internal/networking/networking_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package networking import ( diff --git a/internal/proxy/netproxy.go b/internal/proxy/netproxy.go index 62874ccc..36546cc2 100644 --- a/internal/proxy/netproxy.go +++ b/internal/proxy/netproxy.go @@ -1,790 +1,790 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See LICENSE in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -package proxy - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "math/rand" - "net" - "os" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/go-logr/logr" - - apiv1 "github.com/microsoft/dcp/api/v1" - "github.com/microsoft/dcp/internal/networking" - "github.com/microsoft/dcp/pkg/concurrency" - usvc_io "github.com/microsoft/dcp/pkg/io" - "github.com/microsoft/dcp/pkg/queue" - "github.com/microsoft/dcp/pkg/resiliency" - "github.com/microsoft/dcp/pkg/syncmap" -) - -const ( - // Timeouts used for the read operation. When the read request times out, it gives us the opportunity - // to check for pending write requests and whether the proxy connection should be shut down. - // Reads are interruptible by writes (meaning arriving write will cancel the read operation), - // so the read timeout can be relatively long. - DefaultReadTimeout = 3 * time.Second - - // Timeout used for the write operation. - DefaultWriteTimeout = 5 * time.Second - - // The default connection timeout for establishing a TCP connections. - DefaultConnectionTimeout = 5 * time.Second - - // The maximum number of UDP packets that will be cached for a single client. - MaxCachedUdpPackets = 20 - - // Even though the maximum UDP packet size is 64 kB, most networks have a maximum transmission unit (MTU) - // that is much lower. E.g. Ethernet MTU is only 1500 bytes. And packet fragmentation is something - // that every UDP client must be prepared to deal with. - // That is why we use a buffer of 4kB for UDP packet data. - UdpPacketBufferSize = 4 * 1024 - - // The size of the TCP data buffer, used for single read between two TCP connections (32 kB). - TcpDataBufferSize = 32 * 1024 - - // The time after which a UDP stream will be shut down if it has not been used. - UdpStreamInactivityTimeout = 2 * time.Minute - - // The maximum number of parked TCP connections allowed. - // If exceeded, the oldest parked connection will be closed. - MaxParkedConnections = 20 - - // The maximum buffer size for a parked connection (1 MB). - ParkedConnectionMaxBufferSize = 1024 * 1024 -) - -// A "UDP stream" binds the client with the Endpoint that is serving it. -// Packets sent by the client will be forwarded to the Endpoint using dedicated PacketConn connection. -// Packets received from that PacketConn will be sent back to the client. -type udpStream struct { - clientAddr net.Addr // Address of the client that is being served by this stream - packets *queue.ConcurrentBoundedQueue[[]byte] // Queue of packets from the client - lastUsed *concurrency.AtomicTime // Time when this stream was last used - ctx context.Context // Context that will be cancelled when the stream is to be retired. - cancel context.CancelFunc // The function to cancel the stream context. -} - -// netProxy is an in-process implementation of the Proxy interface that uses -// the standard library's net package to proxy TCP and UDP connections. -type netProxy struct { - mode apiv1.PortProtocol - listenAddress string - listenPort int32 - - effectiveAddress string - effectivePort int32 - - endpointConfigLoadedChannel *concurrency.UnboundedChan[ProxyConfig] - configurationApplied *concurrency.AutoResetEvent - readTimeout time.Duration - writeTimeout time.Duration - connectionTimeout time.Duration - streamSeqNo uint32 - - udpStreams *syncmap.Map[string, udpStream] - - lifetimeCtx context.Context - log logr.Logger - state ProxyState - lock sync.Locker -} - -func NewRuntimeProxy(mode apiv1.PortProtocol, listenAddress string, listenPort int32, lifetimeCtx context.Context, log logr.Logger) Proxy { - return newNetProxy(mode, listenAddress, listenPort, lifetimeCtx, log) -} - -func newNetProxy(mode apiv1.PortProtocol, listenAddress string, listenPort int32, lifetimeCtx context.Context, log logr.Logger) *netProxy { - if mode != apiv1.TCP && mode != apiv1.UDP { - panic(fmt.Errorf("unsupported proxy mode: %s", mode)) - } - - p := netProxy{ - mode: mode, - listenAddress: listenAddress, - listenPort: listenPort, - - endpointConfigLoadedChannel: concurrency.NewUnboundedChan[ProxyConfig](lifetimeCtx), - configurationApplied: concurrency.NewAutoResetEvent(false), - readTimeout: DefaultReadTimeout, - writeTimeout: DefaultWriteTimeout, - connectionTimeout: DefaultConnectionTimeout, - - udpStreams: &syncmap.Map[string, udpStream]{}, - - lifetimeCtx: lifetimeCtx, - log: log, - state: ProxyStateInitial, - lock: &sync.Mutex{}, - } - - return &p -} - -func (p *netProxy) ListenAddress() string { - return p.listenAddress -} - -func (p *netProxy) ListenPort() int32 { - return p.listenPort -} - -func (p *netProxy) EffectiveAddress() string { - return p.effectiveAddress -} - -func (p *netProxy) EffectivePort() int32 { - return p.effectivePort -} - -func (p *netProxy) Start() error { - if p.lifetimeCtx.Err() != nil { - _ = p.setState(ProxyStateAny, ProxyStateFinished) - return fmt.Errorf("proxy cannot be started: lifetime context expired: %w", p.lifetimeCtx.Err()) - } - - if err := p.setState(ProxyStateInitial, ProxyStateRunning); err != nil { - return fmt.Errorf("proxy cannot be started: %w", err) - } - - if p.listenAddress == "" { - p.listenAddress = networking.Localhost - } - - lc := net.ListenConfig{} - switch p.mode { - case apiv1.TCP: - tcpListener, err := lc.Listen(p.lifetimeCtx, "tcp", networking.AddressAndPort(p.listenAddress, p.listenPort)) - if err != nil { - _ = p.setState(ProxyStateAny, ProxyStateFailed) - return err - } - - tcpAddr := tcpListener.Addr().(*net.TCPAddr) - p.effectiveAddress = networking.IpToString(tcpAddr.IP) - p.effectivePort = int32(tcpAddr.Port) - - go p.runTCP(tcpListener) - case apiv1.UDP: - udpListener, err := lc.ListenPacket(p.lifetimeCtx, "udp", networking.AddressAndPort(p.listenAddress, p.listenPort)) - if err != nil { - _ = p.setState(ProxyStateAny, ProxyStateFailed) - return err - } - - udpAddr := udpListener.LocalAddr().(*net.UDPAddr) - p.effectiveAddress = networking.IpToString(udpAddr.IP) - p.effectivePort = int32(udpAddr.Port) - - go p.runUDP(udpListener) - } - - return nil -} - -func (p *netProxy) Configure(newConfig ProxyConfig) error { - state := p.State() - if state == ProxyStateFailed { - return fmt.Errorf("proxy cannot be configured in Failed state") - } - - // Configuration applied after the proxy has finished will not be effective, - // but the call to Configure might come during shutdown, so we do not return an error in that case. - if state != ProxyStateFinished { - p.endpointConfigLoadedChannel.In <- newConfig.Clone() - if p.mode == apiv1.UDP { - p.shutdownAllUDPStreams() - } - if state == ProxyStateRunning { - <-p.configurationApplied.Wait() - } - } - - return nil -} - -func (p *netProxy) State() ProxyState { - p.lock.Lock() - defer p.lock.Unlock() - return p.state -} - -func (p *netProxy) setState(expectedState, newState ProxyState) error { - p.lock.Lock() - defer p.lock.Unlock() - if p.state == newState { - return nil - } - if p.state&expectedState != 0 { - p.state = newState - return nil - } - - return fmt.Errorf("proxy cannot be set to state %s (current state %s)", newState.String(), p.state.String()) -} - -func (p *netProxy) stop(listener io.Closer) { - const errMsg = "Error stopping proxy" - // This Close call will stop TCP Accept / UDP ReadFrom calls, which will ultimately cause the runTCP / runUDP functions to exit - if err := listener.Close(); err != nil { - p.log.Error(err, errMsg) - } - - _ = p.setState(ProxyStateAny, ProxyStateFinished) -} - -// Buffer incoming data from the parked connection while we wait for an endpoint to be configured. -// This allows us to process EOF/FIN while a connection is parked to avoid accumulating partially closed sockets. -type parkedConnection struct { - net.Conn - reader io.ReadCloser -} - -func (c *parkedConnection) Read(b []byte) (int, error) { - return c.reader.Read(b) -} - -func (c *parkedConnection) Close() error { - closeErr := c.Conn.Close() - - closeErr = errors.Join(closeErr, c.reader.Close()) - - return closeErr -} - -func (p *netProxy) runTCP(tcpListener net.Listener) { - var parkedConnections []net.Conn - parkedConnectionMutex := &sync.Mutex{} - - defer func() { - parkedConnectionMutex.Lock() - for _, conn := range parkedConnections { - _ = conn.Close() - } - parkedConnectionMutex.Unlock() - }() - defer p.configurationApplied.SetAndFreeze() // Make sure that Configure() calls return after the proxy has stopped - defer p.stop(tcpListener) - - configVal := &atomic.Value{} // Holds ProxyConfig - // Wait until the config has been loaded the first time before accepting any connections - newConfig := <-p.endpointConfigLoadedChannel.Out - configVal.Store(newConfig) - p.configurationApplied.Set() - p.log.V(1).Info("Initial endpoint configuration loaded", "Config", newConfig.String()) - - // Make a channel that will receive a connection when one is accepted - connectionChannel := concurrency.NewUnboundedChan[net.Conn](p.lifetimeCtx) - go func() { - for { - if p.lifetimeCtx.Err() != nil { - return - } - - // Accept will block until a connection is received or the listener is closed via p.stop() - incoming, err := tcpListener.Accept() - if errors.Is(err, net.ErrClosed) { - // Normal shutdown pathway, don't log - } else if err != nil { - p.log.Info("Error accepting TCP connection", "Error", err) - } else { - connectionChannel.In <- incoming - } - } - }() - - for { - select { - case <-p.lifetimeCtx.Done(): - return - - case newConfig = <-p.endpointConfigLoadedChannel.Out: - if p.lifetimeCtx.Err() != nil { - return - } - configVal.Store(newConfig) - p.configurationApplied.Set() - p.log.V(1).Info("Endpoint configuration changed; new configuration will be applied to future connections...", "Config", newConfig.String()) - - if len(newConfig.Endpoints) > 0 { - parkedConnectionMutex.Lock() - for _, conn := range parkedConnections { - go p.handleTCPConnection(configVal, conn) - } - parkedConnections = nil - parkedConnectionMutex.Unlock() - } - - case incoming := <-connectionChannel.Out: - if p.lifetimeCtx.Err() != nil { - _ = incoming.Close() - return - } - - currentConfig, haveConfig := configVal.Load().(ProxyConfig) - if haveConfig && len(currentConfig.Endpoints) > 0 { - go p.handleTCPConnection(configVal, incoming) - } else { - p.log.V(1).Info("No endpoints configured, parking connection...") - - parkedReader, parkedWriter := usvc_io.NewBufferedPipeWithMaxSize(ParkedConnectionMaxBufferSize) - wrapped := &parkedConnection{Conn: incoming, reader: parkedReader} - - parkedConnectionMutex.Lock() - // If we're at the limit, close the oldest parked connection - if len(parkedConnections) >= MaxParkedConnections { - oldest := parkedConnections[0] - parkedConnections = parkedConnections[1:] - p.log.V(1).Info("Max parked connections reached, closing oldest connection") - go func() { _ = oldest.Close() }() // Close in goroutine to avoid blocking - } - parkedConnections = append(parkedConnections, wrapped) - parkedConnectionMutex.Unlock() - - go func() { - _, copyErr := io.Copy(parkedWriter, incoming) - - parkedConnectionMutex.Lock() - for index, conn := range parkedConnections { - if conn == wrapped { - parkedConnections = append(parkedConnections[:index], parkedConnections[index+1:]...) - break - } - } - parkedConnectionMutex.Unlock() - - // Close the incoming connection when done and remove it from the parked connections list if still present - _ = incoming.Close() - - bufferedWriter := parkedWriter.(*usvc_io.BufferedPipeWriter) - if closeErr := bufferedWriter.CloseWithError(copyErr); closeErr != nil { - p.log.V(1).Info("Error closing parked connection buffer", "Error", closeErr) - } - }() - } - } - } -} - -func (p *netProxy) handleTCPConnection(config *atomic.Value, incoming net.Conn) { - err := resiliency.RetryExponential(p.lifetimeCtx, func() error { - currentConfig, haveConfig := config.Load().(ProxyConfig) - if !haveConfig { - // This should not really happen, as we should have waited until we have SOME config, - // but we can return an error to trigger a retry just in case. - return fmt.Errorf("no configuration available yet") - } - - return p.startTCPStream(incoming, ¤tConfig) - }) - - if err != nil { - p.log.Error(err, "Error handling TCP connection") - } -} - -// Attempts to start a TCP stream between the incoming connection and one of the configured endpoints. -// Normally the returned error results in a retry of the operation. -// Fatal errors should be wrapped in resiliency.Permanent and will not be retried. -// In the current implementation all errors are retry-able. -func (p *netProxy) startTCPStream(incoming net.Conn, config *ProxyConfig) error { - if p.lifetimeCtx.Err() != nil { - _ = incoming.Close() - return nil - } - - endpoint, err := chooseEndpoint(config) - if err != nil { - // Endpoints may become available later. Do not close the incoming connection - return err - } - - if p.log.V(1).Enabled() { - p.log.V(1).Info(fmt.Sprintf("Accepted TCP connection from %s, forwarding to %s ...", - incoming.RemoteAddr().String(), - networking.AddressAndPort(endpoint.Address, endpoint.Port), - )) - } - - var d net.Dialer - // We use relatively short deadline for dialing because we want to try another endpoint fairly quickly if connection establishment fails. - dialContext, dialContextCancel := context.WithTimeout(p.lifetimeCtx, p.connectionTimeout) - defer dialContextCancel() - - ap := networking.AddressAndPort(endpoint.Address, endpoint.Port) - outgoing, dialErr := d.DialContext(dialContext, "tcp", ap) - if dialErr != nil { - p.log.V(1).Info(fmt.Sprintf("Error establishing TCP connection to %s (%s), will try another endpoint", ap, dialErr.Error())) - // Do not close incoming connection - return fmt.Errorf("tried address %s but received the following error: %w", ap, dialErr) - } - - streamCtx, streamCtxCancel := context.WithCancel(p.lifetimeCtx) - streamID := strconv.FormatUint(uint64(atomic.AddUint32(&p.streamSeqNo, 1)), 10) - - if p.log.V(1).Enabled() { - p.log.V(1).Info("Started TCP stream", "Stream", getStreamDescription(streamID, incoming, outgoing)) - } - - go func() { - defer streamCtxCancel() - defer func() { _ = outgoing.Close() }() - defer func() { _ = incoming.Close() }() - - p.runTcpStream(streamID, streamCtx, incoming, outgoing) - - if p.log.V(1).Enabled() { - p.log.V(1).Info(fmt.Sprintf("Closing connections associated with TCP stream %s from %s to %s", - streamID, - incoming.RemoteAddr().String(), - outgoing.RemoteAddr().String()), - ) - } - }() - - return nil -} - -// Copies data from incoming to outgoing connection (and vice versa) until there is no more data to copy, -// or the passed context is cancelled. -// The passed context corresponds to the lifetime of the (bi-directional) stream, -// so if either half of the stream is done/encounters an error, the other half will be stopped. -func (p *netProxy) runTcpStream( - streamID string, - streamCtx context.Context, - incoming net.Conn, - outgoing net.Conn, -) { - // Network stream immediately starts copying data between the two connections. - // It returns when one of the connections is closed, or when an error occurs. - // There are many reasons why a network I/O operation may fail, - // and the failures are reported differently by different APIs and on different platforms. - // That is why we generally do not log errors from those operations except in specific circumstances, - // such as DNS name resolution or initial connection establishment. - // In all other cases we just shut down the communication stream and let the client(s) retry. - - ir, or := StreamNetworkData(streamCtx, incoming, outgoing) - - if p.log.V(1).Enabled() { - silenceErrors := SilenceTcpStreamCompletionErrors.Load() - - if ir.ReadError != nil && !silenceErrors { - p.log.V(1).Error( - ir.ReadError, "The incoming TCP connection encountered a read error", - "Stream", getStreamDescription(streamID, incoming, outgoing), - "Stats", ir.LogProperties(), - ) - } else if ir.WriteError != nil && !silenceErrors { - p.log.V(1).Error( - ir.WriteError, "The incoming TCP connection encountered a write error", - "Stream", getStreamDescription(streamID, incoming, outgoing), - "Stats", ir.LogProperties(), - ) - } else { - p.log.V(1).Info( - "Incoming TCP connection is done", - "Stream", getStreamDescription(streamID, incoming, outgoing), - "Stats", ir.LogProperties(), - ) - } - - if or.WriteError != nil && !silenceErrors { - p.log.V(1).Error( - or.WriteError, "The outgoing TCP connection encountered a write error", - "Stream", getStreamDescription(streamID, incoming, outgoing), - "Stats", or.LogProperties(), - ) - } else if or.ReadError != nil && !silenceErrors { - p.log.V(1).Error( - or.ReadError, "The outgoing TCP connection encountered a read error", - "Stream", getStreamDescription(streamID, incoming, outgoing), - "Stats", or.LogProperties(), - ) - } else { - p.log.V(1).Info( - "Outgoing TCP connection is done", - "Stream", getStreamDescription(streamID, incoming, outgoing), - "Stats", or.LogProperties(), - ) - } - } -} - -func (p *netProxy) runUDP(udpListener net.PacketConn) { - defer p.configurationApplied.SetAndFreeze() // Make sure that Configure() calls return after the proxy has stopped - defer p.stop(udpListener) - defer p.shutdownAllUDPStreams() - - // Wait until the config file has been loaded the first time before accepting any packets - config := <-p.endpointConfigLoadedChannel.Out - p.configurationApplied.Set() - p.log.V(1).Info("Initial endpoint configuration loaded", "Config", config.String()) - - buffer := make([]byte, UdpPacketBufferSize) - - for { - select { - case config = <-p.endpointConfigLoadedChannel.Out: - if p.lifetimeCtx.Err() != nil { - return - } - p.tryStartExistingUDPStreams(config, udpListener) - p.configurationApplied.Set() - p.log.V(1).Info("Configuration changed; new configuration will be applied to future packets...", "Config", config.String()) - - default: - // No config change, continue - } - - if p.lifetimeCtx.Err() != nil { - return - } - - if err := udpListener.SetReadDeadline(time.Now().Add(p.readTimeout)); err != nil { - p.log.Error(err, "Error setting read deadline for proxy UDP connection") - return - } - - // ReadFrom will block for at most 3 seconds before hitting the deadline, at which point it will - // return an error. This is expected so we do not bail out, and as long as the connection context - // is still active, the deadline will be refreshed. - - bytesRead, addr, err := udpListener.ReadFrom(buffer) - if errors.Is(err, os.ErrDeadlineExceeded) { - continue - } else if err != nil { - p.log.Info("Error reading UDP packet from proxy UDP connection", "Error", err) - } else { - q := p.getPacketQueue(addr, udpListener, config) - q.Enqueue(bytes.Clone(buffer[:bytesRead])) - } - } -} - -func (p *netProxy) shutdownAllUDPStreams() { - p.udpStreams.Range(func(clientAddr string, stream udpStream) bool { - p.shutdownUDPStream(stream.clientAddr) - return true - }) -} - -func (p *netProxy) shutdownInactiveUDPStreams() { - p.udpStreams.Range(func(clientAddr string, stream udpStream) bool { - lastUsed := stream.lastUsed.Load() - if time.Since(lastUsed) > UdpStreamInactivityTimeout { - p.shutdownUDPStream(stream.clientAddr) - } - return true - }) -} - -func (p *netProxy) shutdownUDPStream(clientAddr net.Addr) { - stream, loaded := p.udpStreams.LoadAndDelete(clientAddr.String()) - if !loaded { - return - } - - if stream.ctx != nil { - stream.cancel() - // Endpoint streaming goroutine will close the stream listener. - } -} - -func (p *netProxy) tryStartingUDPStream(stream udpStream, proxyConn net.PacketConn, config ProxyConfig) bool { - endpoint, err := chooseEndpoint(&config) - if err != nil { - // No endpoints yet - return false - } - endpointAddr, err := net.ResolveUDPAddr("udp", networking.AddressAndPort(endpoint.Address, endpoint.Port)) - if err != nil { - p.log.Error(err, "Could not resolve endpoint address", "EndpointAddress", endpoint.Address, "EndpointPort", endpoint.Port) - return false - } - - lc := net.ListenConfig{} - ctx, cancel := context.WithCancel(p.lifetimeCtx) - streamListener, err := lc.ListenPacket(ctx, "udp", fmt.Sprintf("%s:", p.listenAddress)) - if err != nil { - p.log.Error(err, "Could not create an endpoint listener for client", "ClientAddr", stream.clientAddr.String()) - cancel() - return false - } - - stream.ctx = ctx - stream.cancel = cancel - p.udpStreams.Store(stream.clientAddr.String(), stream) - - // TODO: log the UDP stream shape: client addr -> proxy conn addr (proxy) stream listener addr -> endpoint addr - go p.streamClientPackets(stream, streamListener, endpointAddr) - go p.streamEndpointPackets(stream, streamListener, proxyConn) - return true -} - -func (p *netProxy) tryStartExistingUDPStreams(config ProxyConfig, proxyConn net.PacketConn) { - p.udpStreams.Range(func(clientAddr string, stream udpStream) bool { - if stream.ctx == nil { - // Stream is not running, try starting it - _ = p.tryStartingUDPStream(stream, proxyConn, config) - } - return true - }) -} - -func (p *netProxy) getPacketQueue(clientAddr net.Addr, proxyConn net.PacketConn, config ProxyConfig) *queue.ConcurrentBoundedQueue[[]byte] { - stream, loaded := p.udpStreams.LoadOrStoreNew(clientAddr.String(), func() udpStream { - return udpStream{ - clientAddr: clientAddr, - packets: queue.NewConcurrentBoundedQueue[[]byte](MaxCachedUdpPackets), - lastUsed: concurrency.AtomicTimeNow(), - ctx: nil, - cancel: nil, - } - }) - - if !loaded || stream.ctx == nil { - _ = p.tryStartingUDPStream(stream, proxyConn, config) - } - - if !loaded { - // Attempt to clean up inactive streams when a new client shows up. - go p.shutdownInactiveUDPStreams() - } - - return stream.packets -} - -func (p *netProxy) streamClientPackets( - stream udpStream, - streamListener net.PacketConn, - endpointAddr *net.UDPAddr, -) { - newPackets := stream.packets.NewData() - - for { - select { - // No need to check the proxy lifetime context because stream context is a child of it. - case <-stream.ctx.Done(): - return - - case <-newPackets: - for { - if stream.ctx.Err() != nil { - return - } - - buf, ok := stream.packets.Dequeue() - if !ok { - break - } - - if err := streamListener.SetWriteDeadline(time.Now().Add(p.writeTimeout)); err != nil { - p.log.V(1).Info("Error setting write deadline for UDP endpoint connection", "Error", err.Error(), "EndpointAddress", endpointAddr.String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - - written, err := streamListener.WriteTo(buf, endpointAddr) - if err != nil { - // Handle write deadline exceeded like any other (unexpected) error for UDP connections - p.log.V(1).Info("Error writing to UDP endpoint connection", "Error", err.Error(), "EndpointAddress", endpointAddr.String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - if written != len(buf) { - p.log.V(1).Info("UDP endpoint connection write did not transmit the whole packet", "BytesSubmitted", len(buf), "BytesWritten", written, "EndpointAddress", endpointAddr.String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - - stream.lastUsed.TryAdvancingTo(time.Now()) - } - } - } -} - -func (p *netProxy) streamEndpointPackets( - stream udpStream, - streamListener net.PacketConn, - proxyConn net.PacketConn, -) { - buffer := make([]byte, UdpPacketBufferSize) - defer streamListener.Close() - - for { - if stream.ctx.Err() != nil { - return - } - - if err := streamListener.SetReadDeadline(time.Now().Add(p.readTimeout)); err != nil { - p.log.V(1).Info("Error setting read deadline for UDP endpoint connection", "Error", err.Error(), "EndpointConnectionAddress", streamListener.LocalAddr().String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - - bytesRead, _, readErr := streamListener.ReadFrom(buffer) - if errors.Is(readErr, os.ErrDeadlineExceeded) { - continue - } else if readErr != nil { - p.log.Info("Error reading UDP packet from proxy UDP connection", "Error", readErr.Error(), "EndpointConnectionAddress", streamListener.LocalAddr().String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - - if err := proxyConn.SetWriteDeadline(time.Now().Add(p.writeTimeout)); err != nil { - p.log.V(1).Info("Error setting write deadline for UDP proxy connection", "Error", err.Error(), "ClientAddress", stream.clientAddr.String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - - written, writeErr := proxyConn.WriteTo(buffer[:bytesRead], stream.clientAddr) - if writeErr != nil { - // Handle write deadline exceeded like any other (unexpected) error for UDP connections - p.log.V(1).Info("Error writing to UDP proxy connection", "Error", writeErr.Error(), "ClientAddress", stream.clientAddr.String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - if written != len(buffer[:bytesRead]) { - p.log.V(1).Info("UDP proxy connection write did not transmit the whole packet", "BytesSubmitted", len(buffer[:bytesRead]), "BytesWritten", written, "ClientAddress", stream.clientAddr.String()) - p.shutdownUDPStream(stream.clientAddr) - return - } - - stream.lastUsed.TryAdvancingTo(time.Now()) - - } -} - -func chooseEndpoint(config *ProxyConfig) (*Endpoint, error) { - // Select a random endpoint from the configured list - if len(config.Endpoints) == 0 { - return nil, errors.New("no endpoints configured") - } - - return &config.Endpoints[rand.Intn(len(config.Endpoints))], nil -} - -func getStreamDescription(streamID string, incoming, outgoing net.Conn) string { - return fmt.Sprintf( - "%s: %s -> %s (proxy) %s -> %s", - streamID, - incoming.RemoteAddr().String(), - incoming.LocalAddr().String(), - outgoing.LocalAddr().String(), - outgoing.RemoteAddr().String(), - ) -} - -var _ Proxy = &netProxy{} +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package proxy + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/rand" + "net" + "os" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/go-logr/logr" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/networking" + "github.com/microsoft/dcp/pkg/concurrency" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/queue" + "github.com/microsoft/dcp/pkg/resiliency" + "github.com/microsoft/dcp/pkg/syncmap" +) + +const ( + // Timeouts used for the read operation. When the read request times out, it gives us the opportunity + // to check for pending write requests and whether the proxy connection should be shut down. + // Reads are interruptible by writes (meaning arriving write will cancel the read operation), + // so the read timeout can be relatively long. + DefaultReadTimeout = 3 * time.Second + + // Timeout used for the write operation. + DefaultWriteTimeout = 5 * time.Second + + // The default connection timeout for establishing a TCP connections. + DefaultConnectionTimeout = 5 * time.Second + + // The maximum number of UDP packets that will be cached for a single client. + MaxCachedUdpPackets = 20 + + // Even though the maximum UDP packet size is 64 kB, most networks have a maximum transmission unit (MTU) + // that is much lower. E.g. Ethernet MTU is only 1500 bytes. And packet fragmentation is something + // that every UDP client must be prepared to deal with. + // That is why we use a buffer of 4kB for UDP packet data. + UdpPacketBufferSize = 4 * 1024 + + // The size of the TCP data buffer, used for single read between two TCP connections (32 kB). + TcpDataBufferSize = 32 * 1024 + + // The time after which a UDP stream will be shut down if it has not been used. + UdpStreamInactivityTimeout = 2 * time.Minute + + // The maximum number of parked TCP connections allowed. + // If exceeded, the oldest parked connection will be closed. + MaxParkedConnections = 20 + + // The maximum buffer size for a parked connection (1 MB). + ParkedConnectionMaxBufferSize = 1024 * 1024 +) + +// A "UDP stream" binds the client with the Endpoint that is serving it. +// Packets sent by the client will be forwarded to the Endpoint using dedicated PacketConn connection. +// Packets received from that PacketConn will be sent back to the client. +type udpStream struct { + clientAddr net.Addr // Address of the client that is being served by this stream + packets *queue.ConcurrentBoundedQueue[[]byte] // Queue of packets from the client + lastUsed *concurrency.AtomicTime // Time when this stream was last used + ctx context.Context // Context that will be cancelled when the stream is to be retired. + cancel context.CancelFunc // The function to cancel the stream context. +} + +// netProxy is an in-process implementation of the Proxy interface that uses +// the standard library's net package to proxy TCP and UDP connections. +type netProxy struct { + mode apiv1.PortProtocol + listenAddress string + listenPort int32 + + effectiveAddress string + effectivePort int32 + + endpointConfigLoadedChannel *concurrency.UnboundedChan[ProxyConfig] + configurationApplied *concurrency.AutoResetEvent + readTimeout time.Duration + writeTimeout time.Duration + connectionTimeout time.Duration + streamSeqNo uint32 + + udpStreams *syncmap.Map[string, udpStream] + + lifetimeCtx context.Context + log logr.Logger + state ProxyState + lock sync.Locker +} + +func NewRuntimeProxy(mode apiv1.PortProtocol, listenAddress string, listenPort int32, lifetimeCtx context.Context, log logr.Logger) Proxy { + return newNetProxy(mode, listenAddress, listenPort, lifetimeCtx, log) +} + +func newNetProxy(mode apiv1.PortProtocol, listenAddress string, listenPort int32, lifetimeCtx context.Context, log logr.Logger) *netProxy { + if mode != apiv1.TCP && mode != apiv1.UDP { + panic(fmt.Errorf("unsupported proxy mode: %s", mode)) + } + + p := netProxy{ + mode: mode, + listenAddress: listenAddress, + listenPort: listenPort, + + endpointConfigLoadedChannel: concurrency.NewUnboundedChan[ProxyConfig](lifetimeCtx), + configurationApplied: concurrency.NewAutoResetEvent(false), + readTimeout: DefaultReadTimeout, + writeTimeout: DefaultWriteTimeout, + connectionTimeout: DefaultConnectionTimeout, + + udpStreams: &syncmap.Map[string, udpStream]{}, + + lifetimeCtx: lifetimeCtx, + log: log, + state: ProxyStateInitial, + lock: &sync.Mutex{}, + } + + return &p +} + +func (p *netProxy) ListenAddress() string { + return p.listenAddress +} + +func (p *netProxy) ListenPort() int32 { + return p.listenPort +} + +func (p *netProxy) EffectiveAddress() string { + return p.effectiveAddress +} + +func (p *netProxy) EffectivePort() int32 { + return p.effectivePort +} + +func (p *netProxy) Start() error { + if p.lifetimeCtx.Err() != nil { + _ = p.setState(ProxyStateAny, ProxyStateFinished) + return fmt.Errorf("proxy cannot be started: lifetime context expired: %w", p.lifetimeCtx.Err()) + } + + if err := p.setState(ProxyStateInitial, ProxyStateRunning); err != nil { + return fmt.Errorf("proxy cannot be started: %w", err) + } + + if p.listenAddress == "" { + p.listenAddress = networking.Localhost + } + + lc := net.ListenConfig{} + switch p.mode { + case apiv1.TCP: + tcpListener, err := lc.Listen(p.lifetimeCtx, "tcp", networking.AddressAndPort(p.listenAddress, p.listenPort)) + if err != nil { + _ = p.setState(ProxyStateAny, ProxyStateFailed) + return err + } + + tcpAddr := tcpListener.Addr().(*net.TCPAddr) + p.effectiveAddress = networking.IpToString(tcpAddr.IP) + p.effectivePort = int32(tcpAddr.Port) + + go p.runTCP(tcpListener) + case apiv1.UDP: + udpListener, err := lc.ListenPacket(p.lifetimeCtx, "udp", networking.AddressAndPort(p.listenAddress, p.listenPort)) + if err != nil { + _ = p.setState(ProxyStateAny, ProxyStateFailed) + return err + } + + udpAddr := udpListener.LocalAddr().(*net.UDPAddr) + p.effectiveAddress = networking.IpToString(udpAddr.IP) + p.effectivePort = int32(udpAddr.Port) + + go p.runUDP(udpListener) + } + + return nil +} + +func (p *netProxy) Configure(newConfig ProxyConfig) error { + state := p.State() + if state == ProxyStateFailed { + return fmt.Errorf("proxy cannot be configured in Failed state") + } + + // Configuration applied after the proxy has finished will not be effective, + // but the call to Configure might come during shutdown, so we do not return an error in that case. + if state != ProxyStateFinished { + p.endpointConfigLoadedChannel.In <- newConfig.Clone() + if p.mode == apiv1.UDP { + p.shutdownAllUDPStreams() + } + if state == ProxyStateRunning { + <-p.configurationApplied.Wait() + } + } + + return nil +} + +func (p *netProxy) State() ProxyState { + p.lock.Lock() + defer p.lock.Unlock() + return p.state +} + +func (p *netProxy) setState(expectedState, newState ProxyState) error { + p.lock.Lock() + defer p.lock.Unlock() + if p.state == newState { + return nil + } + if p.state&expectedState != 0 { + p.state = newState + return nil + } + + return fmt.Errorf("proxy cannot be set to state %s (current state %s)", newState.String(), p.state.String()) +} + +func (p *netProxy) stop(listener io.Closer) { + const errMsg = "Error stopping proxy" + // This Close call will stop TCP Accept / UDP ReadFrom calls, which will ultimately cause the runTCP / runUDP functions to exit + if err := listener.Close(); err != nil { + p.log.Error(err, errMsg) + } + + _ = p.setState(ProxyStateAny, ProxyStateFinished) +} + +// Buffer incoming data from the parked connection while we wait for an endpoint to be configured. +// This allows us to process EOF/FIN while a connection is parked to avoid accumulating partially closed sockets. +type parkedConnection struct { + net.Conn + reader io.ReadCloser +} + +func (c *parkedConnection) Read(b []byte) (int, error) { + return c.reader.Read(b) +} + +func (c *parkedConnection) Close() error { + closeErr := c.Conn.Close() + + closeErr = errors.Join(closeErr, c.reader.Close()) + + return closeErr +} + +func (p *netProxy) runTCP(tcpListener net.Listener) { + var parkedConnections []net.Conn + parkedConnectionMutex := &sync.Mutex{} + + defer func() { + parkedConnectionMutex.Lock() + for _, conn := range parkedConnections { + _ = conn.Close() + } + parkedConnectionMutex.Unlock() + }() + defer p.configurationApplied.SetAndFreeze() // Make sure that Configure() calls return after the proxy has stopped + defer p.stop(tcpListener) + + configVal := &atomic.Value{} // Holds ProxyConfig + // Wait until the config has been loaded the first time before accepting any connections + newConfig := <-p.endpointConfigLoadedChannel.Out + configVal.Store(newConfig) + p.configurationApplied.Set() + p.log.V(1).Info("Initial endpoint configuration loaded", "Config", newConfig.String()) + + // Make a channel that will receive a connection when one is accepted + connectionChannel := concurrency.NewUnboundedChan[net.Conn](p.lifetimeCtx) + go func() { + for { + if p.lifetimeCtx.Err() != nil { + return + } + + // Accept will block until a connection is received or the listener is closed via p.stop() + incoming, err := tcpListener.Accept() + if errors.Is(err, net.ErrClosed) { + // Normal shutdown pathway, don't log + } else if err != nil { + p.log.Info("Error accepting TCP connection", "Error", err) + } else { + connectionChannel.In <- incoming + } + } + }() + + for { + select { + case <-p.lifetimeCtx.Done(): + return + + case newConfig = <-p.endpointConfigLoadedChannel.Out: + if p.lifetimeCtx.Err() != nil { + return + } + configVal.Store(newConfig) + p.configurationApplied.Set() + p.log.V(1).Info("Endpoint configuration changed; new configuration will be applied to future connections...", "Config", newConfig.String()) + + if len(newConfig.Endpoints) > 0 { + parkedConnectionMutex.Lock() + for _, conn := range parkedConnections { + go p.handleTCPConnection(configVal, conn) + } + parkedConnections = nil + parkedConnectionMutex.Unlock() + } + + case incoming := <-connectionChannel.Out: + if p.lifetimeCtx.Err() != nil { + _ = incoming.Close() + return + } + + currentConfig, haveConfig := configVal.Load().(ProxyConfig) + if haveConfig && len(currentConfig.Endpoints) > 0 { + go p.handleTCPConnection(configVal, incoming) + } else { + p.log.V(1).Info("No endpoints configured, parking connection...") + + parkedReader, parkedWriter := usvc_io.NewBufferedPipeWithMaxSize(ParkedConnectionMaxBufferSize) + wrapped := &parkedConnection{Conn: incoming, reader: parkedReader} + + parkedConnectionMutex.Lock() + // If we're at the limit, close the oldest parked connection + if len(parkedConnections) >= MaxParkedConnections { + oldest := parkedConnections[0] + parkedConnections = parkedConnections[1:] + p.log.V(1).Info("Max parked connections reached, closing oldest connection") + go func() { _ = oldest.Close() }() // Close in goroutine to avoid blocking + } + parkedConnections = append(parkedConnections, wrapped) + parkedConnectionMutex.Unlock() + + go func() { + _, copyErr := io.Copy(parkedWriter, incoming) + + parkedConnectionMutex.Lock() + for index, conn := range parkedConnections { + if conn == wrapped { + parkedConnections = append(parkedConnections[:index], parkedConnections[index+1:]...) + break + } + } + parkedConnectionMutex.Unlock() + + // Close the incoming connection when done and remove it from the parked connections list if still present + _ = incoming.Close() + + bufferedWriter := parkedWriter.(*usvc_io.BufferedPipeWriter) + if closeErr := bufferedWriter.CloseWithError(copyErr); closeErr != nil { + p.log.V(1).Info("Error closing parked connection buffer", "Error", closeErr) + } + }() + } + } + } +} + +func (p *netProxy) handleTCPConnection(config *atomic.Value, incoming net.Conn) { + err := resiliency.RetryExponential(p.lifetimeCtx, func() error { + currentConfig, haveConfig := config.Load().(ProxyConfig) + if !haveConfig { + // This should not really happen, as we should have waited until we have SOME config, + // but we can return an error to trigger a retry just in case. + return fmt.Errorf("no configuration available yet") + } + + return p.startTCPStream(incoming, ¤tConfig) + }) + + if err != nil { + p.log.Error(err, "Error handling TCP connection") + } +} + +// Attempts to start a TCP stream between the incoming connection and one of the configured endpoints. +// Normally the returned error results in a retry of the operation. +// Fatal errors should be wrapped in resiliency.Permanent and will not be retried. +// In the current implementation all errors are retry-able. +func (p *netProxy) startTCPStream(incoming net.Conn, config *ProxyConfig) error { + if p.lifetimeCtx.Err() != nil { + _ = incoming.Close() + return nil + } + + endpoint, err := chooseEndpoint(config) + if err != nil { + // Endpoints may become available later. Do not close the incoming connection + return err + } + + if p.log.V(1).Enabled() { + p.log.V(1).Info(fmt.Sprintf("Accepted TCP connection from %s, forwarding to %s ...", + incoming.RemoteAddr().String(), + networking.AddressAndPort(endpoint.Address, endpoint.Port), + )) + } + + var d net.Dialer + // We use relatively short deadline for dialing because we want to try another endpoint fairly quickly if connection establishment fails. + dialContext, dialContextCancel := context.WithTimeout(p.lifetimeCtx, p.connectionTimeout) + defer dialContextCancel() + + ap := networking.AddressAndPort(endpoint.Address, endpoint.Port) + outgoing, dialErr := d.DialContext(dialContext, "tcp", ap) + if dialErr != nil { + p.log.V(1).Info(fmt.Sprintf("Error establishing TCP connection to %s (%s), will try another endpoint", ap, dialErr.Error())) + // Do not close incoming connection + return fmt.Errorf("tried address %s but received the following error: %w", ap, dialErr) + } + + streamCtx, streamCtxCancel := context.WithCancel(p.lifetimeCtx) + streamID := strconv.FormatUint(uint64(atomic.AddUint32(&p.streamSeqNo, 1)), 10) + + if p.log.V(1).Enabled() { + p.log.V(1).Info("Started TCP stream", "Stream", getStreamDescription(streamID, incoming, outgoing)) + } + + go func() { + defer streamCtxCancel() + defer func() { _ = outgoing.Close() }() + defer func() { _ = incoming.Close() }() + + p.runTcpStream(streamID, streamCtx, incoming, outgoing) + + if p.log.V(1).Enabled() { + p.log.V(1).Info(fmt.Sprintf("Closing connections associated with TCP stream %s from %s to %s", + streamID, + incoming.RemoteAddr().String(), + outgoing.RemoteAddr().String()), + ) + } + }() + + return nil +} + +// Copies data from incoming to outgoing connection (and vice versa) until there is no more data to copy, +// or the passed context is cancelled. +// The passed context corresponds to the lifetime of the (bi-directional) stream, +// so if either half of the stream is done/encounters an error, the other half will be stopped. +func (p *netProxy) runTcpStream( + streamID string, + streamCtx context.Context, + incoming net.Conn, + outgoing net.Conn, +) { + // Network stream immediately starts copying data between the two connections. + // It returns when one of the connections is closed, or when an error occurs. + // There are many reasons why a network I/O operation may fail, + // and the failures are reported differently by different APIs and on different platforms. + // That is why we generally do not log errors from those operations except in specific circumstances, + // such as DNS name resolution or initial connection establishment. + // In all other cases we just shut down the communication stream and let the client(s) retry. + + ir, or := StreamNetworkData(streamCtx, incoming, outgoing) + + if p.log.V(1).Enabled() { + silenceErrors := SilenceTcpStreamCompletionErrors.Load() + + if ir.ReadError != nil && !silenceErrors { + p.log.V(1).Error( + ir.ReadError, "The incoming TCP connection encountered a read error", + "Stream", getStreamDescription(streamID, incoming, outgoing), + "Stats", ir.LogProperties(), + ) + } else if ir.WriteError != nil && !silenceErrors { + p.log.V(1).Error( + ir.WriteError, "The incoming TCP connection encountered a write error", + "Stream", getStreamDescription(streamID, incoming, outgoing), + "Stats", ir.LogProperties(), + ) + } else { + p.log.V(1).Info( + "Incoming TCP connection is done", + "Stream", getStreamDescription(streamID, incoming, outgoing), + "Stats", ir.LogProperties(), + ) + } + + if or.WriteError != nil && !silenceErrors { + p.log.V(1).Error( + or.WriteError, "The outgoing TCP connection encountered a write error", + "Stream", getStreamDescription(streamID, incoming, outgoing), + "Stats", or.LogProperties(), + ) + } else if or.ReadError != nil && !silenceErrors { + p.log.V(1).Error( + or.ReadError, "The outgoing TCP connection encountered a read error", + "Stream", getStreamDescription(streamID, incoming, outgoing), + "Stats", or.LogProperties(), + ) + } else { + p.log.V(1).Info( + "Outgoing TCP connection is done", + "Stream", getStreamDescription(streamID, incoming, outgoing), + "Stats", or.LogProperties(), + ) + } + } +} + +func (p *netProxy) runUDP(udpListener net.PacketConn) { + defer p.configurationApplied.SetAndFreeze() // Make sure that Configure() calls return after the proxy has stopped + defer p.stop(udpListener) + defer p.shutdownAllUDPStreams() + + // Wait until the config file has been loaded the first time before accepting any packets + config := <-p.endpointConfigLoadedChannel.Out + p.configurationApplied.Set() + p.log.V(1).Info("Initial endpoint configuration loaded", "Config", config.String()) + + buffer := make([]byte, UdpPacketBufferSize) + + for { + select { + case config = <-p.endpointConfigLoadedChannel.Out: + if p.lifetimeCtx.Err() != nil { + return + } + p.tryStartExistingUDPStreams(config, udpListener) + p.configurationApplied.Set() + p.log.V(1).Info("Configuration changed; new configuration will be applied to future packets...", "Config", config.String()) + + default: + // No config change, continue + } + + if p.lifetimeCtx.Err() != nil { + return + } + + if err := udpListener.SetReadDeadline(time.Now().Add(p.readTimeout)); err != nil { + p.log.Error(err, "Error setting read deadline for proxy UDP connection") + return + } + + // ReadFrom will block for at most 3 seconds before hitting the deadline, at which point it will + // return an error. This is expected so we do not bail out, and as long as the connection context + // is still active, the deadline will be refreshed. + + bytesRead, addr, err := udpListener.ReadFrom(buffer) + if errors.Is(err, os.ErrDeadlineExceeded) { + continue + } else if err != nil { + p.log.Info("Error reading UDP packet from proxy UDP connection", "Error", err) + } else { + q := p.getPacketQueue(addr, udpListener, config) + q.Enqueue(bytes.Clone(buffer[:bytesRead])) + } + } +} + +func (p *netProxy) shutdownAllUDPStreams() { + p.udpStreams.Range(func(clientAddr string, stream udpStream) bool { + p.shutdownUDPStream(stream.clientAddr) + return true + }) +} + +func (p *netProxy) shutdownInactiveUDPStreams() { + p.udpStreams.Range(func(clientAddr string, stream udpStream) bool { + lastUsed := stream.lastUsed.Load() + if time.Since(lastUsed) > UdpStreamInactivityTimeout { + p.shutdownUDPStream(stream.clientAddr) + } + return true + }) +} + +func (p *netProxy) shutdownUDPStream(clientAddr net.Addr) { + stream, loaded := p.udpStreams.LoadAndDelete(clientAddr.String()) + if !loaded { + return + } + + if stream.ctx != nil { + stream.cancel() + // Endpoint streaming goroutine will close the stream listener. + } +} + +func (p *netProxy) tryStartingUDPStream(stream udpStream, proxyConn net.PacketConn, config ProxyConfig) bool { + endpoint, err := chooseEndpoint(&config) + if err != nil { + // No endpoints yet + return false + } + endpointAddr, err := net.ResolveUDPAddr("udp", networking.AddressAndPort(endpoint.Address, endpoint.Port)) + if err != nil { + p.log.Error(err, "Could not resolve endpoint address", "EndpointAddress", endpoint.Address, "EndpointPort", endpoint.Port) + return false + } + + lc := net.ListenConfig{} + ctx, cancel := context.WithCancel(p.lifetimeCtx) + streamListener, err := lc.ListenPacket(ctx, "udp", fmt.Sprintf("%s:", p.listenAddress)) + if err != nil { + p.log.Error(err, "Could not create an endpoint listener for client", "ClientAddr", stream.clientAddr.String()) + cancel() + return false + } + + stream.ctx = ctx + stream.cancel = cancel + p.udpStreams.Store(stream.clientAddr.String(), stream) + + // TODO: log the UDP stream shape: client addr -> proxy conn addr (proxy) stream listener addr -> endpoint addr + go p.streamClientPackets(stream, streamListener, endpointAddr) + go p.streamEndpointPackets(stream, streamListener, proxyConn) + return true +} + +func (p *netProxy) tryStartExistingUDPStreams(config ProxyConfig, proxyConn net.PacketConn) { + p.udpStreams.Range(func(clientAddr string, stream udpStream) bool { + if stream.ctx == nil { + // Stream is not running, try starting it + _ = p.tryStartingUDPStream(stream, proxyConn, config) + } + return true + }) +} + +func (p *netProxy) getPacketQueue(clientAddr net.Addr, proxyConn net.PacketConn, config ProxyConfig) *queue.ConcurrentBoundedQueue[[]byte] { + stream, loaded := p.udpStreams.LoadOrStoreNew(clientAddr.String(), func() udpStream { + return udpStream{ + clientAddr: clientAddr, + packets: queue.NewConcurrentBoundedQueue[[]byte](MaxCachedUdpPackets), + lastUsed: concurrency.AtomicTimeNow(), + ctx: nil, + cancel: nil, + } + }) + + if !loaded || stream.ctx == nil { + _ = p.tryStartingUDPStream(stream, proxyConn, config) + } + + if !loaded { + // Attempt to clean up inactive streams when a new client shows up. + go p.shutdownInactiveUDPStreams() + } + + return stream.packets +} + +func (p *netProxy) streamClientPackets( + stream udpStream, + streamListener net.PacketConn, + endpointAddr *net.UDPAddr, +) { + newPackets := stream.packets.NewData() + + for { + select { + // No need to check the proxy lifetime context because stream context is a child of it. + case <-stream.ctx.Done(): + return + + case <-newPackets: + for { + if stream.ctx.Err() != nil { + return + } + + buf, ok := stream.packets.Dequeue() + if !ok { + break + } + + if err := streamListener.SetWriteDeadline(time.Now().Add(p.writeTimeout)); err != nil { + p.log.V(1).Info("Error setting write deadline for UDP endpoint connection", "Error", err.Error(), "EndpointAddress", endpointAddr.String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + + written, err := streamListener.WriteTo(buf, endpointAddr) + if err != nil { + // Handle write deadline exceeded like any other (unexpected) error for UDP connections + p.log.V(1).Info("Error writing to UDP endpoint connection", "Error", err.Error(), "EndpointAddress", endpointAddr.String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + if written != len(buf) { + p.log.V(1).Info("UDP endpoint connection write did not transmit the whole packet", "BytesSubmitted", len(buf), "BytesWritten", written, "EndpointAddress", endpointAddr.String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + + stream.lastUsed.TryAdvancingTo(time.Now()) + } + } + } +} + +func (p *netProxy) streamEndpointPackets( + stream udpStream, + streamListener net.PacketConn, + proxyConn net.PacketConn, +) { + buffer := make([]byte, UdpPacketBufferSize) + defer streamListener.Close() + + for { + if stream.ctx.Err() != nil { + return + } + + if err := streamListener.SetReadDeadline(time.Now().Add(p.readTimeout)); err != nil { + p.log.V(1).Info("Error setting read deadline for UDP endpoint connection", "Error", err.Error(), "EndpointConnectionAddress", streamListener.LocalAddr().String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + + bytesRead, _, readErr := streamListener.ReadFrom(buffer) + if errors.Is(readErr, os.ErrDeadlineExceeded) { + continue + } else if readErr != nil { + p.log.Info("Error reading UDP packet from proxy UDP connection", "Error", readErr.Error(), "EndpointConnectionAddress", streamListener.LocalAddr().String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + + if err := proxyConn.SetWriteDeadline(time.Now().Add(p.writeTimeout)); err != nil { + p.log.V(1).Info("Error setting write deadline for UDP proxy connection", "Error", err.Error(), "ClientAddress", stream.clientAddr.String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + + written, writeErr := proxyConn.WriteTo(buffer[:bytesRead], stream.clientAddr) + if writeErr != nil { + // Handle write deadline exceeded like any other (unexpected) error for UDP connections + p.log.V(1).Info("Error writing to UDP proxy connection", "Error", writeErr.Error(), "ClientAddress", stream.clientAddr.String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + if written != len(buffer[:bytesRead]) { + p.log.V(1).Info("UDP proxy connection write did not transmit the whole packet", "BytesSubmitted", len(buffer[:bytesRead]), "BytesWritten", written, "ClientAddress", stream.clientAddr.String()) + p.shutdownUDPStream(stream.clientAddr) + return + } + + stream.lastUsed.TryAdvancingTo(time.Now()) + + } +} + +func chooseEndpoint(config *ProxyConfig) (*Endpoint, error) { + // Select a random endpoint from the configured list + if len(config.Endpoints) == 0 { + return nil, errors.New("no endpoints configured") + } + + return &config.Endpoints[rand.Intn(len(config.Endpoints))], nil +} + +func getStreamDescription(streamID string, incoming, outgoing net.Conn) string { + return fmt.Sprintf( + "%s: %s -> %s (proxy) %s -> %s", + streamID, + incoming.RemoteAddr().String(), + incoming.LocalAddr().String(), + outgoing.LocalAddr().String(), + outgoing.RemoteAddr().String(), + ) +} + +var _ Proxy = &netProxy{} diff --git a/internal/proxy/network_stream_unix.go b/internal/proxy/network_stream_unix.go index efa20cf7..3dec0202 100644 --- a/internal/proxy/network_stream_unix.go +++ b/internal/proxy/network_stream_unix.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package proxy import ( diff --git a/internal/proxy/network_stream_windows.go b/internal/proxy/network_stream_windows.go index ba692451..814f52a9 100644 --- a/internal/proxy/network_stream_windows.go +++ b/internal/proxy/network_stream_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package proxy import ( diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 00b288a2..e8ff0057 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -1,863 +1,863 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See LICENSE in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -package proxy - -import ( - "bytes" - "context" - "errors" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "slices" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/go-logr/logr" - "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/util/wait" - - apiv1 "github.com/microsoft/dcp/api/v1" - "github.com/microsoft/dcp/internal/networking" - "github.com/microsoft/dcp/pkg/logger" - "github.com/microsoft/dcp/pkg/osutil" - "github.com/microsoft/dcp/pkg/testutil" -) - -var ( - log = logger.New("proxy-tests").Logger -) - -func isTimeoutError(err error) bool { - if err == nil { - return false - } - if errors.Is(err, os.ErrDeadlineExceeded) { - return true - } - var netErr net.Error - return errors.As(err, &netErr) && netErr.Timeout() -} - -func TestMain(m *testing.M) { - networking.EnableStrictMruPortHandling(log) - code := m.Run() - os.Exit(code) -} - -func TestInvalidProxyMode(t *testing.T) { - t.Parallel() - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - require.Panics(t, func() { newNetProxy("tcp", "", 0, ctx, logr.Discard()) }) -} - -func TestTCPModeProxy(t *testing.T) { - t.Parallel() - - // Set up a server - serverListener, port := setupTcpServer(t, autoAllocatePort) - - // Set up a proxy - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Feed the config to the proxy - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - err := proxy.Configure(config) - require.NoError(t, err) - - // Verify the proxy sends data to the server - verifiyProxiedTcpConnection(t, proxy, serverListener) - - // Clean up - require.NoError(t, serverListener.Close()) -} - -func TestTCPConfigurationChange(t *testing.T) { - t.Parallel() - - // Set up two servers - listener1, port1 := setupTcpServer(t, autoAllocatePort) - listener2, port2 := setupTcpServer(t, autoAllocatePort) - - // Set up a proxy - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Feed config to the proxy, targeting the first server - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port1)}, - }, - } - err := proxy.Configure(config) - require.NoError(t, err) - - // Verify the proxy sends data to the fist server - verifiyProxiedTcpConnection(t, proxy, listener1) - - // Change configuration to target second server - config = ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port2)}, - }, - } - err = proxy.Configure(config) - require.NoError(t, err) - - // Verify the proxy sends data to the second server - verifiyProxiedTcpConnection(t, proxy, listener2) - - // Clean up - require.NoError(t, listener1.Close()) - require.NoError(t, listener2.Close()) -} - -func TestTCPClientCanSendDataBeforeEndpointsExist(t *testing.T) { - t.Parallel() - - // Set up a proxy - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - proxy.connectionTimeout = 500 * time.Millisecond - - // Write some data to the proxy - clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) - require.NoError(t, err) - require.NotNil(t, clientConn) - msg := []byte("This is amazing!") - _, err = clientConn.Write(msg) - require.NoError(t, err) - - // Configure the proxy with no endpoints several times - const emptyConfigTries = 3 - emptyConfig := ProxyConfig{ - Endpoints: []Endpoint{}, - } - for i := 0; i < emptyConfigTries; i++ { - err = proxy.Configure(emptyConfig) - require.NoError(t, err) - } - - // Configure the proxy with valid endpoint - port, err := networking.GetFreePort(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, log) - require.NoError(t, err) - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - err = proxy.Configure(config) - require.NoError(t, err) - - // Set up a server - serverListener, _ := setupTcpServer(t, port) - - // Verify the proxy sends data to the server - proxiedConnection, err := serverListener.Accept() - require.NoError(t, err) - require.NotNil(t, proxiedConnection) - - // Verify the message - buffer := make([]byte, len(msg)) - _, err = proxiedConnection.Read(buffer) - require.NoError(t, err) - require.EqualValues(t, msg, buffer) - - // Clean up - require.NoError(t, proxiedConnection.Close()) - require.NoError(t, clientConn.Close()) - require.NoError(t, serverListener.Close()) -} - -func TestUDPModeProxy(t *testing.T) { - t.Parallel() - - // Set up a server - serverConn, port := setupUdpServer(t, autoAllocatePort) - - // Set up a proxy - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - proxy, logSink := setupUdpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Feed the config to the proxy - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - err := proxy.Configure(config) - require.NoError(t, err) - - // Verify data can be sent both ways - verifyProxiedUdpConnection(t, proxy, serverConn) - - // Clean up - require.NoError(t, serverConn.Close()) -} - -func TestUDPTwoEndpoints(t *testing.T) { - t.Parallel() - - // Set up server - serverConn, port := setupUdpServer(t, autoAllocatePort) - - // Set up a proxy - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - proxy, logSink := setupUdpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Feed the config to the proxy - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - err := proxy.Configure(config) - require.NoError(t, err) - - // Set up two clients - clientConn1, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, 0)) - require.NoError(t, err) - require.NotNil(t, clientConn1) - clientConn2, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, 0)) - require.NoError(t, err) - require.NotNil(t, clientConn2) - - // We should be able to send data back and forth with these two clients independently - verifyUDPPacketFlow(t, proxy, serverConn, clientConn1) - verifyUDPPacketFlow(t, proxy, serverConn, clientConn2) - verifyUDPPacketFlow(t, proxy, serverConn, clientConn1) - verifyUDPPacketFlow(t, proxy, serverConn, clientConn2) - - // Clean up - require.NoError(t, clientConn1.Close()) - require.NoError(t, clientConn2.Close()) - require.NoError(t, serverConn.Close()) -} - -func TestTCPParkedConnectionMaxBufferSize(t *testing.T) { - t.Parallel() - - // Set up a proxy with no endpoints configured - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Configure the proxy with no endpoints to force connection parking - emptyConfig := ProxyConfig{ - Endpoints: []Endpoint{}, - } - err := proxy.Configure(emptyConfig) - require.NoError(t, err) - - // Connect a client to the proxy - clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) - require.NoError(t, err) - require.NotNil(t, clientConn) - defer clientConn.Close() - - // The buffer max size is 1MB (1024 * 1024 bytes) - const maxBufferSize = 1024 * 1024 - const chunkSize = 64 * 1024 // 64KB chunks - chunk := make([]byte, chunkSize) - - const maxWriteBytes = 32 * 1024 * 1024 - const writeTimeout = 100 * time.Millisecond - - var totalWritten atomic.Int64 - timedOut := false - - for totalWritten.Load() < maxWriteBytes { - setDeadlineErr := clientConn.SetWriteDeadline(time.Now().Add(writeTimeout)) - require.NoError(t, setDeadlineErr) - - n, writeErr := clientConn.Write(chunk) - totalWritten.Add(int64(n)) - if writeErr != nil { - if isTimeoutError(writeErr) { - timedOut = true - break - } - t.Fatalf("Unexpected write error after %d bytes: %v", totalWritten.Load(), writeErr) - } - } - - written := totalWritten.Load() - require.True(t, timedOut, "Expected writes to block before %d bytes", maxWriteBytes) - require.Greater(t, written, int64(maxBufferSize/2), - "Should have written a substantial amount before blocking") - require.Less(t, written, int64(maxWriteBytes), - "Should have blocked before writing %d bytes", maxWriteBytes) -} - -func TestTCPParkedConnectionClosedBeforeEndpointConfigured(t *testing.T) { - t.Parallel() - - // Set up a proxy with no endpoints configured - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Configure the proxy with no endpoints to force connection parking - emptyConfig := ProxyConfig{ - Endpoints: []Endpoint{}, - } - err := proxy.Configure(emptyConfig) - require.NoError(t, err) - - // Connect a client to the proxy and send some data - clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) - require.NoError(t, err) - require.NotNil(t, clientConn) - - // Send some data while no endpoints are configured - testData := []byte("data that should not be received") - bytesWritten, writeErr := clientConn.Write(testData) - require.NoError(t, writeErr) - require.Equal(t, len(testData), bytesWritten) - - // Close the client connection before configuring an endpoint - tcpConn, ok := clientConn.(*net.TCPConn) - require.True(t, ok) - require.NoError(t, tcpConn.CloseWrite()) - - // Wait for the proxy to observe the close and shut down the parked connection - waitErr := wait.PollUntilContextTimeout(context.Background(), 25*time.Millisecond, 5*time.Second, true, func(_ context.Context) (bool, error) { - setReadDeadlineErr := clientConn.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) - if setReadDeadlineErr != nil { - return true, nil - } - buffer := make([]byte, 1) - _, readErr := clientConn.Read(buffer) - if readErr == nil { - return false, nil - } - if isTimeoutError(readErr) { - return false, nil - } - return true, nil - }) - require.NoError(t, waitErr) - require.NoError(t, clientConn.Close()) - - // Now set up a server - serverListener, port := setupTcpServer(t, autoAllocatePort) - defer serverListener.Close() - - // Configure the proxy with the new endpoint - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - configErr := proxy.Configure(config) - require.NoError(t, configErr) - - // Try to accept a connection - there should be none since the client closed before endpoint was configured - setDeadlineErr := serverListener.(*net.TCPListener).SetDeadline(time.Now().Add(500 * time.Millisecond)) - require.NoError(t, setDeadlineErr) - serverConn, acceptErr := serverListener.Accept() - - // We expect either no connection (timeout) or a connection that immediately closes with no data - if acceptErr != nil { - // Timeout is expected - no connection should be forwarded - require.True(t, errors.Is(acceptErr, os.ErrDeadlineExceeded), - "Expected timeout error, got: %v", acceptErr) - } else { - // If we got a connection, it should have no data and close immediately - defer serverConn.Close() - setReadDeadlineErr := serverConn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) - require.NoError(t, setReadDeadlineErr) - buffer := make([]byte, 1024) - n, readErr := serverConn.Read(buffer) - require.Equal(t, 0, n, "Should not have received any data from a closed parked connection") - require.Error(t, readErr, "Should get an error reading from a closed connection") - } -} - -func TestTCPParkedConnectionMaxCountLimit(t *testing.T) { - t.Parallel() - - // Set up a proxy with no endpoints configured - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - // Use logr.Discard() instead of mock logger since this test causes expected TCP errors - // when connections are closed due to the limit - proxy := newNetProxy(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.Discard()) - require.NotNil(t, proxy) - err := proxy.Start() - require.NoError(t, err) - - // Configure the proxy with no endpoints to force connection parking - emptyConfig := ProxyConfig{ - Endpoints: []Endpoint{}, - } - err = proxy.Configure(emptyConfig) - require.NoError(t, err) - - // Create more connections than the max allowed (MaxParkedConnections = 20) - const numConnections = 25 - connections := make([]net.Conn, numConnections) - - for i := 0; i < numConnections; i++ { - conn, dialErr := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) - require.NoError(t, dialErr) - require.NotNil(t, conn) - connections[i] = conn - - // Write a unique identifier for each connection - msg := fmt.Sprintf("connection-%02d", i) - _, writeErr := conn.Write([]byte(msg)) - require.NoError(t, writeErr) - } - - // The first 5 connections (0-4) should have been closed due to the limit - // Try to write to them - should fail - for i := 0; i < numConnections-MaxParkedConnections; i++ { - conn := connections[i] - waitErr := wait.PollUntilContextTimeout(context.Background(), 25*time.Millisecond, 5*time.Second, true, func(_ context.Context) (bool, error) { - setWriteDeadlineErr := conn.SetWriteDeadline(time.Now().Add(200 * time.Millisecond)) - if setWriteDeadlineErr != nil { - return true, nil - } - _, writeErr := conn.Write([]byte("test")) - setReadDeadlineErr := conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) - if setReadDeadlineErr != nil { - return true, nil - } - buf := make([]byte, 1) - _, readErr := conn.Read(buf) - - writeClosed := writeErr != nil && !isTimeoutError(writeErr) - readClosed := readErr != nil && !isTimeoutError(readErr) - return writeClosed || readClosed, nil - }) - require.NoError(t, waitErr) - } - - // The remaining connections (5-24) should still be valid - // Configure an endpoint and verify they can send data - serverListener, port := setupTcpServer(t, autoAllocatePort) - defer serverListener.Close() - - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - configErr := proxy.Configure(config) - require.NoError(t, configErr) - - // Count how many connections are accepted by the server - acceptedCount := 0 - setDeadlineErr := serverListener.(*net.TCPListener).SetDeadline(time.Now().Add(2 * time.Second)) - require.NoError(t, setDeadlineErr) - for { - serverConn, acceptErr := serverListener.Accept() - if acceptErr != nil { - break - } - acceptedCount++ - serverConn.Close() - } - - // Should have accepted exactly MaxParkedConnections (20) connections - require.Equal(t, MaxParkedConnections, acceptedCount, - "Should have exactly %d parked connections after limit enforcement", MaxParkedConnections) - - // Clean up remaining client connections - for _, conn := range connections { - _ = conn.Close() - } -} - -func TestRandomEndpointSelection(t *testing.T) { - t.Parallel() - - const tries = 20 - - ctx, cancelFunc := context.WithCancel(context.Background()) - proxy := newNetProxy(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.Discard()) - require.NotNil(t, proxy) - defer cancelFunc() - - config := &ProxyConfig{ - Endpoints: []Endpoint{ - {Address: "127.0.0.1", Port: 8080}, - {Address: "127.0.0.2", Port: 8081}, - }, - } - - haveSeenZeroOne := false - haveSeenZeroTwo := false - - for range make([]any, tries) { - endpoint, err := chooseEndpoint(config) - require.NoError(t, err) - require.NotNil(t, endpoint) - switch endpoint.Address { - case "127.0.0.1": - haveSeenZeroOne = true - case "127.0.0.2": - haveSeenZeroTwo = true - } - } - - require.True(t, haveSeenZeroOne) - require.True(t, haveSeenZeroTwo) - - config = &ProxyConfig{} - endpoint, err := chooseEndpoint(config) - require.Error(t, err) - require.Nil(t, endpoint) -} - -// Run 100 simultaneous clients making requests to a single server behind the proxy -// until the context is cancelled (after 30 seconds). -// The test verifies that the proxy can handle at least 10 000 connections -// (with single, successful request round-trip) during that time. -func TestTCPProxyConnectionThroughput(t *testing.T) { - testutil.SkipIfNotEnableAdvancedNetworking(t) - - t.Parallel() - - // Set up a TCP server that echoes back data and closes the connection - serverListener, port := setupTcpServer(t, autoAllocatePort) - defer serverListener.Close() - - var successfulRequests atomic.Int32 - - // Start a goroutine to handle client connections - go func() { - for { - conn, err := serverListener.Accept() - if err != nil { - // Server closed, exit the goroutine - return - } - - go func(c net.Conn) { - defer c.Close() - - // Read data from client - buffer := make([]byte, 1024) - n, readErr := c.Read(buffer) - if readErr != nil { - return - } - - // Echo the data back - _, _ = c.Write(buffer[:n]) - }(conn) - } - }() - - // Setup test context - ctx, cancelFunc := testutil.GetTestContext(t, 30*time.Second) - defer cancelFunc() - - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - // Feed the config to the proxy - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, - }, - } - err := proxy.Configure(config) - require.NoError(t, err) - - // Run 100 simultaneous clients making requests until the context is cancelled - const numUsers = 100 - var wg sync.WaitGroup - wg.Add(numUsers) - - errorsChan := make(chan error, numUsers) - - clientsCtx, clientsCancel := context.WithTimeout(ctx, 10*time.Second) - defer clientsCancel() - - // Function to run a single client connection - runUser := func(runCtx context.Context) { - defer wg.Done() - - runClient := func() error { - // Set up a client connection to the proxy - clientConn, dialErr := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) - if dialErr != nil { - return fmt.Errorf("failed to connect to proxy: %w", dialErr) - } - defer clientConn.Close() - - clientStop := context.AfterFunc(runCtx, func() { - clientConn.Close() - }) - defer clientStop() - - // Generate 500 bytes of random data - randomData := []byte(testutil.GetRandLetters(t, 500)) - - // Send the data to the proxy - _, writeErr := clientConn.Write(randomData) - if writeErr != nil { - return fmt.Errorf("failed to write to proxy: %w", writeErr) - } - - // Read the response - responseData := make([]byte, len(randomData)) - _, readErr := clientConn.Read(responseData) - if readErr != nil { - return fmt.Errorf("failed to read from proxy: %w", readErr) - } - - // Verify the received data matches what was sent - if !slices.Equal(randomData, responseData) { - return fmt.Errorf("data mismatch: sent %d bytes, received %d bytes with different content", - len(randomData), len(responseData)) - } - - successfulRequests.Add(1) - - return nil - } - - // Run the client in a loop until the context is cancelled or an error occurs - for { - if runCtx.Err() != nil { - // Context was cancelled, exit the loop - return - } - - clientErr := runClient() - if clientErr != nil { - if runCtx.Err() == nil { - // Report the error if the context hasn't been cancelled - errorsChan <- clientErr - } - return - } - } - } - - // Launch all clients in parallel - for i := 0; i < numUsers; i++ { - go runUser(clientsCtx) - } - - // Wait for all clients to finish - wg.Wait() - close(errorsChan) - - // Check for errors - var clientErr error - for err := range errorsChan { - clientErr = errors.Join(clientErr, err) - } - - require.NoError(t, clientErr, "Error running client connections") - - t.Logf("Successful requests: %d", successfulRequests.Load()) - require.Greater(t, successfulRequests.Load(), int32(10_000), "Client throughput is less than expected") -} - -// This test connects a .NET server and client via the proxy in order to reproduce a specific issue with continuous streams -// in older DCP releases. In versions before v0.9.2, the data received by the client would eventually be corrupted. -func TestTCPProxyContinuousStream(t *testing.T) { - testutil.SkipIfNotEnableAdvancedNetworking(t) - - t.Parallel() - - serverPort, portErr := networking.GetFreePort(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, log) - require.NoError(t, portErr, "Failed to get free port for server") - - // Set up a proxy - ctx, cancelFunc := context.WithTimeout(context.Background(), 35*time.Second) - defer cancelFunc() - proxy, logSink := setupTcpProxy(t, ctx) - defer logSink.AssertNotCalled(t, "Error") - - rootDir, findRootErr := osutil.FindRootFor(osutil.DirTarget, "test") - require.NoError(t, findRootErr, "Failed to find root directory for ./test") - - go func() { - // If the server exits due to an error, cancel the context - defer cancelFunc() - - serverCmd := exec.CommandContext(ctx, "dotnet", "run") - serverCmd.Dir = filepath.Join(rootDir, "test", "HttpContentStreamRepro.Server") - serverCmd.Env = os.Environ() - serverCmd.Env = append(serverCmd.Env, fmt.Sprintf("ASPNETCORE_URLS=http://localhost:%d", serverPort)) - serverErr := serverCmd.Run() - if ctx.Err() == nil { - // If the context is not cancelled, make sure the server didn't return an error - require.NoError(t, serverErr, "Failed running server") - } - }() - - // Feed the config to the proxy - config := ProxyConfig{ - Endpoints: []Endpoint{ - {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(serverPort)}, - }, - } - err := proxy.Configure(config) - require.NoError(t, err) - - clientCmd := exec.CommandContext(ctx, "dotnet", "run") - clientCmd.Dir = filepath.Join(rootDir, "test", "HttpContentStreamRepro.Client") - clientCmd.Env = os.Environ() - clientCmd.Env = append(clientCmd.Env, fmt.Sprintf("SERVER_URL=http://%s", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort()))) - clientCmd.Stderr = os.Stderr - clientErr := clientCmd.Run() - - if ctx.Err() == nil { - // If the context is not cancelled, make sure the client didn't return an error - require.NoError(t, clientErr) - } -} - -const autoAllocatePort = 0 - -func setupTcpServer(t *testing.T, port int32) (net.Listener, int) { - serverListener, err := net.Listen("tcp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, port)) - require.NoError(t, err) - require.NotNil(t, serverListener) - effectivePort := serverListener.Addr().(*net.TCPAddr).Port - require.Greater(t, effectivePort, 0) - return serverListener, effectivePort -} - -func setupTcpProxy(t *testing.T, ctx context.Context) (*netProxy, *testutil.MockLoggerSink) { - logSink := testutil.NewMockLoggerSink() - proxy := newNetProxy(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.New(logSink)) - require.NotNil(t, proxy) - - err := proxy.Start() - require.NoError(t, err) - require.Equal(t, networking.IPv4LocalhostDefaultAddress, proxy.EffectiveAddress()) - require.Greater(t, proxy.EffectivePort(), int32(0)) - return proxy, logSink -} - -func verifiyProxiedTcpConnection(t *testing.T, proxy *netProxy, server net.Listener) { - // Set up a client - clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) - require.NoError(t, err) - require.NotNil(t, clientConn) - - // Have the client send a message - msg := []byte(testutil.GetRandLetters(t, 6)) - _, err = clientConn.Write(msg) - require.NoError(t, err) - - // Have the server receive a message - proxiedConnection, err := server.Accept() - require.NoError(t, err) - require.NotNil(t, proxiedConnection) - - // Verify the message - buffer := make([]byte, len(msg)) - _, err = proxiedConnection.Read(buffer) - require.NoError(t, err) - require.EqualValues(t, msg, buffer, "Message received by server is not the same as the message sent by the client") - - require.NoError(t, proxiedConnection.Close()) - require.NoError(t, clientConn.Close()) -} - -func setupUdpServer(t *testing.T, port int32) (net.PacketConn, int) { - serverConn, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, port)) - require.NoError(t, err) - require.NotNil(t, serverConn) - effectivePort := serverConn.LocalAddr().(*net.UDPAddr).Port - require.Greater(t, effectivePort, 0) - return serverConn, effectivePort -} - -func setupUdpProxy(t *testing.T, ctx context.Context) (*netProxy, *testutil.MockLoggerSink) { - logSink := testutil.NewMockLoggerSink() - proxy := newNetProxy(apiv1.UDP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.New(logSink)) - require.NotNil(t, proxy) - - err := proxy.Start() - require.NoError(t, err) - require.Equal(t, networking.IPv4LocalhostDefaultAddress, proxy.EffectiveAddress()) - require.Greater(t, proxy.EffectivePort(), int32(0)) - return proxy, logSink -} - -func verifyProxiedUdpConnection(t *testing.T, proxy *netProxy, serverConn net.PacketConn) { - // Set up a client - clientConn, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, 0)) - require.NoError(t, err) - require.NotNil(t, clientConn) - - verifyUDPPacketFlow(t, proxy, serverConn, clientConn) - - // Clean up - require.NoError(t, clientConn.Close()) -} - -func verifyUDPPacketFlow( - t *testing.T, - proxy *netProxy, - serverConn net.PacketConn, - clientConn net.PacketConn, -) { - // Have the client send a message - msg := []byte(testutil.GetRandLetters(t, 6)) - n, err := clientConn.WriteTo(msg, &net.UDPAddr{IP: net.ParseIP(proxy.EffectiveAddress()), Port: int(proxy.EffectivePort())}) - require.NoError(t, err) - require.Equal(t, len(msg), n) - - // Have the server receive a message - buffer := make([]byte, len(msg)) - n, clientAddr, err := serverConn.ReadFrom(buffer) - require.NoError(t, err) - require.Equal(t, len(msg), n) - // Note that the client addr will be, in fact, the proxy address/port - - // Verify the message - require.True(t, bytes.Equal(buffer, msg), "Message received by the server ('%s') is not the same as the message sent by the client ('%s')", string(buffer), string(msg)) - - // Have the server send a message back - msg = []byte(testutil.GetRandLetters(t, 6)) - n, err = serverConn.WriteTo(msg, clientAddr) - require.NoError(t, err) - require.Equal(t, len(msg), n) - - // Have the client receive the response message - buffer = make([]byte, len(msg)) - n, proxyAddr, err := clientConn.ReadFrom(buffer) - require.NoError(t, err) - require.Equal(t, len(msg), n) - require.Equal(t, proxy.EffectiveAddress(), proxyAddr.(*net.UDPAddr).IP.String()) - require.Equal(t, proxy.EffectivePort(), int32(proxyAddr.(*net.UDPAddr).Port)) - require.True(t, bytes.Equal(buffer, msg), "Response message received by the client ('%s') is not the same as the message sent by the server ('%s')", string(buffer), string(msg)) -} +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package proxy + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/wait" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/networking" + "github.com/microsoft/dcp/pkg/logger" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/testutil" +) + +var ( + log = logger.New("proxy-tests").Logger +) + +func isTimeoutError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, os.ErrDeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +func TestMain(m *testing.M) { + networking.EnableStrictMruPortHandling(log) + code := m.Run() + os.Exit(code) +} + +func TestInvalidProxyMode(t *testing.T) { + t.Parallel() + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + require.Panics(t, func() { newNetProxy("tcp", "", 0, ctx, logr.Discard()) }) +} + +func TestTCPModeProxy(t *testing.T) { + t.Parallel() + + // Set up a server + serverListener, port := setupTcpServer(t, autoAllocatePort) + + // Set up a proxy + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Feed the config to the proxy + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + err := proxy.Configure(config) + require.NoError(t, err) + + // Verify the proxy sends data to the server + verifiyProxiedTcpConnection(t, proxy, serverListener) + + // Clean up + require.NoError(t, serverListener.Close()) +} + +func TestTCPConfigurationChange(t *testing.T) { + t.Parallel() + + // Set up two servers + listener1, port1 := setupTcpServer(t, autoAllocatePort) + listener2, port2 := setupTcpServer(t, autoAllocatePort) + + // Set up a proxy + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Feed config to the proxy, targeting the first server + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port1)}, + }, + } + err := proxy.Configure(config) + require.NoError(t, err) + + // Verify the proxy sends data to the fist server + verifiyProxiedTcpConnection(t, proxy, listener1) + + // Change configuration to target second server + config = ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port2)}, + }, + } + err = proxy.Configure(config) + require.NoError(t, err) + + // Verify the proxy sends data to the second server + verifiyProxiedTcpConnection(t, proxy, listener2) + + // Clean up + require.NoError(t, listener1.Close()) + require.NoError(t, listener2.Close()) +} + +func TestTCPClientCanSendDataBeforeEndpointsExist(t *testing.T) { + t.Parallel() + + // Set up a proxy + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + proxy.connectionTimeout = 500 * time.Millisecond + + // Write some data to the proxy + clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) + require.NoError(t, err) + require.NotNil(t, clientConn) + msg := []byte("This is amazing!") + _, err = clientConn.Write(msg) + require.NoError(t, err) + + // Configure the proxy with no endpoints several times + const emptyConfigTries = 3 + emptyConfig := ProxyConfig{ + Endpoints: []Endpoint{}, + } + for i := 0; i < emptyConfigTries; i++ { + err = proxy.Configure(emptyConfig) + require.NoError(t, err) + } + + // Configure the proxy with valid endpoint + port, err := networking.GetFreePort(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, log) + require.NoError(t, err) + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + err = proxy.Configure(config) + require.NoError(t, err) + + // Set up a server + serverListener, _ := setupTcpServer(t, port) + + // Verify the proxy sends data to the server + proxiedConnection, err := serverListener.Accept() + require.NoError(t, err) + require.NotNil(t, proxiedConnection) + + // Verify the message + buffer := make([]byte, len(msg)) + _, err = proxiedConnection.Read(buffer) + require.NoError(t, err) + require.EqualValues(t, msg, buffer) + + // Clean up + require.NoError(t, proxiedConnection.Close()) + require.NoError(t, clientConn.Close()) + require.NoError(t, serverListener.Close()) +} + +func TestUDPModeProxy(t *testing.T) { + t.Parallel() + + // Set up a server + serverConn, port := setupUdpServer(t, autoAllocatePort) + + // Set up a proxy + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + proxy, logSink := setupUdpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Feed the config to the proxy + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + err := proxy.Configure(config) + require.NoError(t, err) + + // Verify data can be sent both ways + verifyProxiedUdpConnection(t, proxy, serverConn) + + // Clean up + require.NoError(t, serverConn.Close()) +} + +func TestUDPTwoEndpoints(t *testing.T) { + t.Parallel() + + // Set up server + serverConn, port := setupUdpServer(t, autoAllocatePort) + + // Set up a proxy + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + proxy, logSink := setupUdpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Feed the config to the proxy + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + err := proxy.Configure(config) + require.NoError(t, err) + + // Set up two clients + clientConn1, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, 0)) + require.NoError(t, err) + require.NotNil(t, clientConn1) + clientConn2, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, 0)) + require.NoError(t, err) + require.NotNil(t, clientConn2) + + // We should be able to send data back and forth with these two clients independently + verifyUDPPacketFlow(t, proxy, serverConn, clientConn1) + verifyUDPPacketFlow(t, proxy, serverConn, clientConn2) + verifyUDPPacketFlow(t, proxy, serverConn, clientConn1) + verifyUDPPacketFlow(t, proxy, serverConn, clientConn2) + + // Clean up + require.NoError(t, clientConn1.Close()) + require.NoError(t, clientConn2.Close()) + require.NoError(t, serverConn.Close()) +} + +func TestTCPParkedConnectionMaxBufferSize(t *testing.T) { + t.Parallel() + + // Set up a proxy with no endpoints configured + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Configure the proxy with no endpoints to force connection parking + emptyConfig := ProxyConfig{ + Endpoints: []Endpoint{}, + } + err := proxy.Configure(emptyConfig) + require.NoError(t, err) + + // Connect a client to the proxy + clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) + require.NoError(t, err) + require.NotNil(t, clientConn) + defer clientConn.Close() + + // The buffer max size is 1MB (1024 * 1024 bytes) + const maxBufferSize = 1024 * 1024 + const chunkSize = 64 * 1024 // 64KB chunks + chunk := make([]byte, chunkSize) + + const maxWriteBytes = 32 * 1024 * 1024 + const writeTimeout = 100 * time.Millisecond + + var totalWritten atomic.Int64 + timedOut := false + + for totalWritten.Load() < maxWriteBytes { + setDeadlineErr := clientConn.SetWriteDeadline(time.Now().Add(writeTimeout)) + require.NoError(t, setDeadlineErr) + + n, writeErr := clientConn.Write(chunk) + totalWritten.Add(int64(n)) + if writeErr != nil { + if isTimeoutError(writeErr) { + timedOut = true + break + } + t.Fatalf("Unexpected write error after %d bytes: %v", totalWritten.Load(), writeErr) + } + } + + written := totalWritten.Load() + require.True(t, timedOut, "Expected writes to block before %d bytes", maxWriteBytes) + require.Greater(t, written, int64(maxBufferSize/2), + "Should have written a substantial amount before blocking") + require.Less(t, written, int64(maxWriteBytes), + "Should have blocked before writing %d bytes", maxWriteBytes) +} + +func TestTCPParkedConnectionClosedBeforeEndpointConfigured(t *testing.T) { + t.Parallel() + + // Set up a proxy with no endpoints configured + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Configure the proxy with no endpoints to force connection parking + emptyConfig := ProxyConfig{ + Endpoints: []Endpoint{}, + } + err := proxy.Configure(emptyConfig) + require.NoError(t, err) + + // Connect a client to the proxy and send some data + clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) + require.NoError(t, err) + require.NotNil(t, clientConn) + + // Send some data while no endpoints are configured + testData := []byte("data that should not be received") + bytesWritten, writeErr := clientConn.Write(testData) + require.NoError(t, writeErr) + require.Equal(t, len(testData), bytesWritten) + + // Close the client connection before configuring an endpoint + tcpConn, ok := clientConn.(*net.TCPConn) + require.True(t, ok) + require.NoError(t, tcpConn.CloseWrite()) + + // Wait for the proxy to observe the close and shut down the parked connection + waitErr := wait.PollUntilContextTimeout(context.Background(), 25*time.Millisecond, 5*time.Second, true, func(_ context.Context) (bool, error) { + setReadDeadlineErr := clientConn.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + if setReadDeadlineErr != nil { + return true, nil + } + buffer := make([]byte, 1) + _, readErr := clientConn.Read(buffer) + if readErr == nil { + return false, nil + } + if isTimeoutError(readErr) { + return false, nil + } + return true, nil + }) + require.NoError(t, waitErr) + require.NoError(t, clientConn.Close()) + + // Now set up a server + serverListener, port := setupTcpServer(t, autoAllocatePort) + defer serverListener.Close() + + // Configure the proxy with the new endpoint + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + configErr := proxy.Configure(config) + require.NoError(t, configErr) + + // Try to accept a connection - there should be none since the client closed before endpoint was configured + setDeadlineErr := serverListener.(*net.TCPListener).SetDeadline(time.Now().Add(500 * time.Millisecond)) + require.NoError(t, setDeadlineErr) + serverConn, acceptErr := serverListener.Accept() + + // We expect either no connection (timeout) or a connection that immediately closes with no data + if acceptErr != nil { + // Timeout is expected - no connection should be forwarded + require.True(t, errors.Is(acceptErr, os.ErrDeadlineExceeded), + "Expected timeout error, got: %v", acceptErr) + } else { + // If we got a connection, it should have no data and close immediately + defer serverConn.Close() + setReadDeadlineErr := serverConn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) + require.NoError(t, setReadDeadlineErr) + buffer := make([]byte, 1024) + n, readErr := serverConn.Read(buffer) + require.Equal(t, 0, n, "Should not have received any data from a closed parked connection") + require.Error(t, readErr, "Should get an error reading from a closed connection") + } +} + +func TestTCPParkedConnectionMaxCountLimit(t *testing.T) { + t.Parallel() + + // Set up a proxy with no endpoints configured + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + // Use logr.Discard() instead of mock logger since this test causes expected TCP errors + // when connections are closed due to the limit + proxy := newNetProxy(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.Discard()) + require.NotNil(t, proxy) + err := proxy.Start() + require.NoError(t, err) + + // Configure the proxy with no endpoints to force connection parking + emptyConfig := ProxyConfig{ + Endpoints: []Endpoint{}, + } + err = proxy.Configure(emptyConfig) + require.NoError(t, err) + + // Create more connections than the max allowed (MaxParkedConnections = 20) + const numConnections = 25 + connections := make([]net.Conn, numConnections) + + for i := 0; i < numConnections; i++ { + conn, dialErr := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) + require.NoError(t, dialErr) + require.NotNil(t, conn) + connections[i] = conn + + // Write a unique identifier for each connection + msg := fmt.Sprintf("connection-%02d", i) + _, writeErr := conn.Write([]byte(msg)) + require.NoError(t, writeErr) + } + + // The first 5 connections (0-4) should have been closed due to the limit + // Try to write to them - should fail + for i := 0; i < numConnections-MaxParkedConnections; i++ { + conn := connections[i] + waitErr := wait.PollUntilContextTimeout(context.Background(), 25*time.Millisecond, 5*time.Second, true, func(_ context.Context) (bool, error) { + setWriteDeadlineErr := conn.SetWriteDeadline(time.Now().Add(200 * time.Millisecond)) + if setWriteDeadlineErr != nil { + return true, nil + } + _, writeErr := conn.Write([]byte("test")) + setReadDeadlineErr := conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + if setReadDeadlineErr != nil { + return true, nil + } + buf := make([]byte, 1) + _, readErr := conn.Read(buf) + + writeClosed := writeErr != nil && !isTimeoutError(writeErr) + readClosed := readErr != nil && !isTimeoutError(readErr) + return writeClosed || readClosed, nil + }) + require.NoError(t, waitErr) + } + + // The remaining connections (5-24) should still be valid + // Configure an endpoint and verify they can send data + serverListener, port := setupTcpServer(t, autoAllocatePort) + defer serverListener.Close() + + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + configErr := proxy.Configure(config) + require.NoError(t, configErr) + + // Count how many connections are accepted by the server + acceptedCount := 0 + setDeadlineErr := serverListener.(*net.TCPListener).SetDeadline(time.Now().Add(2 * time.Second)) + require.NoError(t, setDeadlineErr) + for { + serverConn, acceptErr := serverListener.Accept() + if acceptErr != nil { + break + } + acceptedCount++ + serverConn.Close() + } + + // Should have accepted exactly MaxParkedConnections (20) connections + require.Equal(t, MaxParkedConnections, acceptedCount, + "Should have exactly %d parked connections after limit enforcement", MaxParkedConnections) + + // Clean up remaining client connections + for _, conn := range connections { + _ = conn.Close() + } +} + +func TestRandomEndpointSelection(t *testing.T) { + t.Parallel() + + const tries = 20 + + ctx, cancelFunc := context.WithCancel(context.Background()) + proxy := newNetProxy(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.Discard()) + require.NotNil(t, proxy) + defer cancelFunc() + + config := &ProxyConfig{ + Endpoints: []Endpoint{ + {Address: "127.0.0.1", Port: 8080}, + {Address: "127.0.0.2", Port: 8081}, + }, + } + + haveSeenZeroOne := false + haveSeenZeroTwo := false + + for range make([]any, tries) { + endpoint, err := chooseEndpoint(config) + require.NoError(t, err) + require.NotNil(t, endpoint) + switch endpoint.Address { + case "127.0.0.1": + haveSeenZeroOne = true + case "127.0.0.2": + haveSeenZeroTwo = true + } + } + + require.True(t, haveSeenZeroOne) + require.True(t, haveSeenZeroTwo) + + config = &ProxyConfig{} + endpoint, err := chooseEndpoint(config) + require.Error(t, err) + require.Nil(t, endpoint) +} + +// Run 100 simultaneous clients making requests to a single server behind the proxy +// until the context is cancelled (after 30 seconds). +// The test verifies that the proxy can handle at least 10 000 connections +// (with single, successful request round-trip) during that time. +func TestTCPProxyConnectionThroughput(t *testing.T) { + testutil.SkipIfNotEnableAdvancedNetworking(t) + + t.Parallel() + + // Set up a TCP server that echoes back data and closes the connection + serverListener, port := setupTcpServer(t, autoAllocatePort) + defer serverListener.Close() + + var successfulRequests atomic.Int32 + + // Start a goroutine to handle client connections + go func() { + for { + conn, err := serverListener.Accept() + if err != nil { + // Server closed, exit the goroutine + return + } + + go func(c net.Conn) { + defer c.Close() + + // Read data from client + buffer := make([]byte, 1024) + n, readErr := c.Read(buffer) + if readErr != nil { + return + } + + // Echo the data back + _, _ = c.Write(buffer[:n]) + }(conn) + } + }() + + // Setup test context + ctx, cancelFunc := testutil.GetTestContext(t, 30*time.Second) + defer cancelFunc() + + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + // Feed the config to the proxy + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(port)}, + }, + } + err := proxy.Configure(config) + require.NoError(t, err) + + // Run 100 simultaneous clients making requests until the context is cancelled + const numUsers = 100 + var wg sync.WaitGroup + wg.Add(numUsers) + + errorsChan := make(chan error, numUsers) + + clientsCtx, clientsCancel := context.WithTimeout(ctx, 10*time.Second) + defer clientsCancel() + + // Function to run a single client connection + runUser := func(runCtx context.Context) { + defer wg.Done() + + runClient := func() error { + // Set up a client connection to the proxy + clientConn, dialErr := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) + if dialErr != nil { + return fmt.Errorf("failed to connect to proxy: %w", dialErr) + } + defer clientConn.Close() + + clientStop := context.AfterFunc(runCtx, func() { + clientConn.Close() + }) + defer clientStop() + + // Generate 500 bytes of random data + randomData := []byte(testutil.GetRandLetters(t, 500)) + + // Send the data to the proxy + _, writeErr := clientConn.Write(randomData) + if writeErr != nil { + return fmt.Errorf("failed to write to proxy: %w", writeErr) + } + + // Read the response + responseData := make([]byte, len(randomData)) + _, readErr := clientConn.Read(responseData) + if readErr != nil { + return fmt.Errorf("failed to read from proxy: %w", readErr) + } + + // Verify the received data matches what was sent + if !slices.Equal(randomData, responseData) { + return fmt.Errorf("data mismatch: sent %d bytes, received %d bytes with different content", + len(randomData), len(responseData)) + } + + successfulRequests.Add(1) + + return nil + } + + // Run the client in a loop until the context is cancelled or an error occurs + for { + if runCtx.Err() != nil { + // Context was cancelled, exit the loop + return + } + + clientErr := runClient() + if clientErr != nil { + if runCtx.Err() == nil { + // Report the error if the context hasn't been cancelled + errorsChan <- clientErr + } + return + } + } + } + + // Launch all clients in parallel + for i := 0; i < numUsers; i++ { + go runUser(clientsCtx) + } + + // Wait for all clients to finish + wg.Wait() + close(errorsChan) + + // Check for errors + var clientErr error + for err := range errorsChan { + clientErr = errors.Join(clientErr, err) + } + + require.NoError(t, clientErr, "Error running client connections") + + t.Logf("Successful requests: %d", successfulRequests.Load()) + require.Greater(t, successfulRequests.Load(), int32(10_000), "Client throughput is less than expected") +} + +// This test connects a .NET server and client via the proxy in order to reproduce a specific issue with continuous streams +// in older DCP releases. In versions before v0.9.2, the data received by the client would eventually be corrupted. +func TestTCPProxyContinuousStream(t *testing.T) { + testutil.SkipIfNotEnableAdvancedNetworking(t) + + t.Parallel() + + serverPort, portErr := networking.GetFreePort(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, log) + require.NoError(t, portErr, "Failed to get free port for server") + + // Set up a proxy + ctx, cancelFunc := context.WithTimeout(context.Background(), 35*time.Second) + defer cancelFunc() + proxy, logSink := setupTcpProxy(t, ctx) + defer logSink.AssertNotCalled(t, "Error") + + rootDir, findRootErr := osutil.FindRootFor(osutil.DirTarget, "test") + require.NoError(t, findRootErr, "Failed to find root directory for ./test") + + go func() { + // If the server exits due to an error, cancel the context + defer cancelFunc() + + serverCmd := exec.CommandContext(ctx, "dotnet", "run") + serverCmd.Dir = filepath.Join(rootDir, "test", "HttpContentStreamRepro.Server") + serverCmd.Env = os.Environ() + serverCmd.Env = append(serverCmd.Env, fmt.Sprintf("ASPNETCORE_URLS=http://localhost:%d", serverPort)) + serverErr := serverCmd.Run() + if ctx.Err() == nil { + // If the context is not cancelled, make sure the server didn't return an error + require.NoError(t, serverErr, "Failed running server") + } + }() + + // Feed the config to the proxy + config := ProxyConfig{ + Endpoints: []Endpoint{ + {Address: networking.IPv4LocalhostDefaultAddress, Port: int32(serverPort)}, + }, + } + err := proxy.Configure(config) + require.NoError(t, err) + + clientCmd := exec.CommandContext(ctx, "dotnet", "run") + clientCmd.Dir = filepath.Join(rootDir, "test", "HttpContentStreamRepro.Client") + clientCmd.Env = os.Environ() + clientCmd.Env = append(clientCmd.Env, fmt.Sprintf("SERVER_URL=http://%s", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort()))) + clientCmd.Stderr = os.Stderr + clientErr := clientCmd.Run() + + if ctx.Err() == nil { + // If the context is not cancelled, make sure the client didn't return an error + require.NoError(t, clientErr) + } +} + +const autoAllocatePort = 0 + +func setupTcpServer(t *testing.T, port int32) (net.Listener, int) { + serverListener, err := net.Listen("tcp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, port)) + require.NoError(t, err) + require.NotNil(t, serverListener) + effectivePort := serverListener.Addr().(*net.TCPAddr).Port + require.Greater(t, effectivePort, 0) + return serverListener, effectivePort +} + +func setupTcpProxy(t *testing.T, ctx context.Context) (*netProxy, *testutil.MockLoggerSink) { + logSink := testutil.NewMockLoggerSink() + proxy := newNetProxy(apiv1.TCP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.New(logSink)) + require.NotNil(t, proxy) + + err := proxy.Start() + require.NoError(t, err) + require.Equal(t, networking.IPv4LocalhostDefaultAddress, proxy.EffectiveAddress()) + require.Greater(t, proxy.EffectivePort(), int32(0)) + return proxy, logSink +} + +func verifiyProxiedTcpConnection(t *testing.T, proxy *netProxy, server net.Listener) { + // Set up a client + clientConn, err := net.Dial("tcp", networking.AddressAndPort(proxy.EffectiveAddress(), proxy.EffectivePort())) + require.NoError(t, err) + require.NotNil(t, clientConn) + + // Have the client send a message + msg := []byte(testutil.GetRandLetters(t, 6)) + _, err = clientConn.Write(msg) + require.NoError(t, err) + + // Have the server receive a message + proxiedConnection, err := server.Accept() + require.NoError(t, err) + require.NotNil(t, proxiedConnection) + + // Verify the message + buffer := make([]byte, len(msg)) + _, err = proxiedConnection.Read(buffer) + require.NoError(t, err) + require.EqualValues(t, msg, buffer, "Message received by server is not the same as the message sent by the client") + + require.NoError(t, proxiedConnection.Close()) + require.NoError(t, clientConn.Close()) +} + +func setupUdpServer(t *testing.T, port int32) (net.PacketConn, int) { + serverConn, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, port)) + require.NoError(t, err) + require.NotNil(t, serverConn) + effectivePort := serverConn.LocalAddr().(*net.UDPAddr).Port + require.Greater(t, effectivePort, 0) + return serverConn, effectivePort +} + +func setupUdpProxy(t *testing.T, ctx context.Context) (*netProxy, *testutil.MockLoggerSink) { + logSink := testutil.NewMockLoggerSink() + proxy := newNetProxy(apiv1.UDP, networking.IPv4LocalhostDefaultAddress, 0, ctx, logr.New(logSink)) + require.NotNil(t, proxy) + + err := proxy.Start() + require.NoError(t, err) + require.Equal(t, networking.IPv4LocalhostDefaultAddress, proxy.EffectiveAddress()) + require.Greater(t, proxy.EffectivePort(), int32(0)) + return proxy, logSink +} + +func verifyProxiedUdpConnection(t *testing.T, proxy *netProxy, serverConn net.PacketConn) { + // Set up a client + clientConn, err := net.ListenPacket("udp", networking.AddressAndPort(networking.IPv4LocalhostDefaultAddress, 0)) + require.NoError(t, err) + require.NotNil(t, clientConn) + + verifyUDPPacketFlow(t, proxy, serverConn, clientConn) + + // Clean up + require.NoError(t, clientConn.Close()) +} + +func verifyUDPPacketFlow( + t *testing.T, + proxy *netProxy, + serverConn net.PacketConn, + clientConn net.PacketConn, +) { + // Have the client send a message + msg := []byte(testutil.GetRandLetters(t, 6)) + n, err := clientConn.WriteTo(msg, &net.UDPAddr{IP: net.ParseIP(proxy.EffectiveAddress()), Port: int(proxy.EffectivePort())}) + require.NoError(t, err) + require.Equal(t, len(msg), n) + + // Have the server receive a message + buffer := make([]byte, len(msg)) + n, clientAddr, err := serverConn.ReadFrom(buffer) + require.NoError(t, err) + require.Equal(t, len(msg), n) + // Note that the client addr will be, in fact, the proxy address/port + + // Verify the message + require.True(t, bytes.Equal(buffer, msg), "Message received by the server ('%s') is not the same as the message sent by the client ('%s')", string(buffer), string(msg)) + + // Have the server send a message back + msg = []byte(testutil.GetRandLetters(t, 6)) + n, err = serverConn.WriteTo(msg, clientAddr) + require.NoError(t, err) + require.Equal(t, len(msg), n) + + // Have the client receive the response message + buffer = make([]byte, len(msg)) + n, proxyAddr, err := clientConn.ReadFrom(buffer) + require.NoError(t, err) + require.Equal(t, len(msg), n) + require.Equal(t, proxy.EffectiveAddress(), proxyAddr.(*net.UDPAddr).IP.String()) + require.Equal(t, proxy.EffectivePort(), int32(proxyAddr.(*net.UDPAddr).Port)) + require.True(t, bytes.Equal(buffer, msg), "Response message received by the client ('%s') is not the same as the message sent by the server ('%s')", string(buffer), string(msg)) +} diff --git a/internal/statestore/lease_owner.go b/internal/statestore/lease_owner.go new file mode 100644 index 00000000..b65e5105 --- /dev/null +++ b/internal/statestore/lease_owner.go @@ -0,0 +1,62 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package statestore + +import ( + "fmt" + + "github.com/microsoft/dcp/pkg/process" +) + +func CurrentResourceLeaseOwner() (process.ProcessTreeItem, error) { + currentProcess, currentProcessErr := process.This() + if currentProcessErr != nil { + return process.ProcessTreeItem{}, fmt.Errorf("could not get current process identity: %w", currentProcessErr) + } + + return normalizeResourceLeaseOwner(currentProcess) +} + +func normalizeResourceLeaseOwner(owner process.ProcessTreeItem) (process.ProcessTreeItem, error) { + if owner.Pid <= 0 { + return process.ProcessTreeItem{}, fmt.Errorf("resource lease owner process ID must be positive") + } + if owner.IdentityTime.IsZero() { + return process.ProcessTreeItem{}, fmt.Errorf("resource lease owner process identity time cannot be zero") + } + + return process.ProcessTreeItem{ + Pid: owner.Pid, + IdentityTime: owner.IdentityTime.UTC(), + }, nil +} + +func resourceLeaseOwnerFromDB(ownerPID int64, ownerIdentityTime string) (process.ProcessTreeItem, error) { + pid, pidConvertErr := process.Int64_ToPidT(ownerPID) + if pidConvertErr != nil { + return process.ProcessTreeItem{}, fmt.Errorf("could not convert resource lease owner process ID: %w", pidConvertErr) + } + identityTime, identityTimeErr := timeFromString(ownerIdentityTime) + if identityTimeErr != nil { + return process.ProcessTreeItem{}, fmt.Errorf("could not parse resource lease owner process identity time: %w", identityTimeErr) + } + + owner := process.ProcessTreeItem{ + Pid: pid, + IdentityTime: identityTime, + } + return normalizeResourceLeaseOwner(owner) +} + +func resourceLeaseOwnerIsActive(owner process.ProcessTreeItem) bool { + normalizedOwner, normalizeErr := normalizeResourceLeaseOwner(owner) + if normalizeErr != nil { + return false + } + + _, findErr := process.FindProcess(normalizedOwner.Pid, normalizedOwner.IdentityTime) + return findErr == nil +} diff --git a/internal/statestore/leases.go b/internal/statestore/leases.go new file mode 100644 index 00000000..93a9ec76 --- /dev/null +++ b/internal/statestore/leases.go @@ -0,0 +1,380 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package statestore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/microsoft/dcp/pkg/process" +) + +var ( + ErrResourceLeaseHeld = errors.New("resource lease is held by another owner") + ErrResourceLeaseNotHeld = errors.New("resource lease is not held by owner") +) + +type ResourceLease struct { + ResourceKey string + OwnerProcess process.ProcessTreeItem + UpdatedAt time.Time +} + +type ResourceLeaseHeldError struct { + Lease ResourceLease +} + +func (e *ResourceLeaseHeldError) Error() string { + return fmt.Sprintf("%s: %s", ErrResourceLeaseHeld, e.Lease.ResourceKey) +} + +func (e *ResourceLeaseHeldError) Unwrap() error { + return ErrResourceLeaseHeld +} + +func HeldResourceLease(err error) (*ResourceLease, bool) { + var heldErr *ResourceLeaseHeldError + if !errors.As(err, &heldErr) { + return nil, false + } + + lease := heldErr.Lease + return &lease, true +} + +type LeasableResource interface { + GetLeaseKey() string +} + +func resourceLeaseKey(resource LeasableResource) (string, error) { + if resource == nil { + return "", fmt.Errorf("%w: resource cannot be nil", ErrInvalidArgument) + } + resourceKey := strings.TrimSpace(resource.GetLeaseKey()) + if resourceKey == "" { + return "", fmt.Errorf("%w: resource lease key cannot be empty", ErrInvalidArgument) + } + return resourceKey, nil +} + +func (s *Store) AcquireResourceLease(ctx context.Context, resource LeasableResource, ownerProcess process.ProcessTreeItem, revalidationInterval time.Duration) (*ResourceLease, error) { + resourceKey, resourceKeyErr := resourceLeaseKey(resource) + if resourceKeyErr != nil { + return nil, resourceKeyErr + } + normalizedOwner, ownerErr := normalizeResourceLeaseOwner(ownerProcess) + if ownerErr != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidArgument, ownerErr) + } + if revalidationInterval <= 0 { + return nil, fmt.Errorf("%w: lease revalidation interval must be positive", ErrInvalidArgument) + } + + now := time.Now().UTC() + ownerPID := normalizedOwner.Pid + ownerIdentityTime := timeString(normalizedOwner.IdentityTime) + var lease *ResourceLease + + txErr := s.withImmediateTx(ctx, func(conn *sql.Conn) error { + insertResult, insertErr := conn.ExecContext( + ctx, + `INSERT OR IGNORE INTO resource_locks(resource_key, owner_pid, owner_identity_time, updated_at_unix_nano) + VALUES(?, ?, ?, ?)`, + resourceKey, + ownerPID, + ownerIdentityTime, + unixNano(now), + ) + if insertErr != nil { + return fmt.Errorf("could not acquire resource lease '%s': %w", resourceKey, insertErr) + } + + rowsAffected, rowsErr := insertResult.RowsAffected() + if rowsErr != nil { + return fmt.Errorf("could not confirm resource lease acquisition for '%s': %w", resourceKey, rowsErr) + } + if rowsAffected == 1 { + var getErr error + lease, getErr = getResourceLease(ctx, conn, resourceKey) + return getErr + } + + existingLease, getErr := getResourceLease(ctx, conn, resourceKey) + if getErr != nil { + return getErr + } + sameOwner := existingLease.OwnerProcess.Pid == normalizedOwner.Pid && + existingLease.OwnerProcess.IdentityTime.Equal(normalizedOwner.IdentityTime) + if !sameOwner && now.Sub(existingLease.UpdatedAt) < revalidationInterval { + return &ResourceLeaseHeldError{Lease: *existingLease} + } + if !sameOwner && resourceLeaseOwnerIsActive(existingLease.OwnerProcess) { + updateErr := updateResourceLeaseTimestamp(ctx, conn, resourceKey, existingLease, now) + if updateErr != nil { + return updateErr + } + return &ResourceLeaseHeldError{Lease: *existingLease} + } + + updateErr := updateResourceLeaseOwner(ctx, conn, resourceKey, normalizedOwner, now) + if updateErr != nil { + return updateErr + } + lease, getErr = getResourceLease(ctx, conn, resourceKey) + return getErr + }) + if txErr != nil { + return nil, txErr + } + + return lease, nil +} + +func (s *Store) ReleaseResourceLease(ctx context.Context, resource LeasableResource, ownerProcess process.ProcessTreeItem) error { + resourceKey, resourceKeyErr := resourceLeaseKey(resource) + if resourceKeyErr != nil { + return resourceKeyErr + } + normalizedOwner, ownerErr := normalizeResourceLeaseOwner(ownerProcess) + if ownerErr != nil { + return fmt.Errorf("%w: %w", ErrInvalidArgument, ownerErr) + } + ownerPID := normalizedOwner.Pid + ownerIdentityTime := timeString(normalizedOwner.IdentityTime) + + return s.withImmediateTx(ctx, func(conn *sql.Conn) error { + result, execErr := conn.ExecContext( + ctx, + `DELETE FROM resource_locks + WHERE resource_key = ? AND owner_pid = ? AND owner_identity_time = ?`, + resourceKey, + ownerPID, + ownerIdentityTime, + ) + if execErr != nil { + return fmt.Errorf("could not release resource lease '%s': %w", resourceKey, execErr) + } + rowsAffected, rowsErr := result.RowsAffected() + if rowsErr != nil { + return fmt.Errorf("could not confirm resource lease release for '%s': %w", resourceKey, rowsErr) + } + if rowsAffected == 0 { + return fmt.Errorf("%w: %s", ErrResourceLeaseNotHeld, resourceKey) + } + + return nil + }) +} + +func (s *Store) VerifyResourceLeaseHeld(ctx context.Context, resource LeasableResource, ownerProcess process.ProcessTreeItem) error { + resourceKey, resourceKeyErr := resourceLeaseKey(resource) + if resourceKeyErr != nil { + return resourceKeyErr + } + normalizedOwner, ownerErr := normalizeResourceLeaseOwner(ownerProcess) + if ownerErr != nil { + return fmt.Errorf("%w: %w", ErrInvalidArgument, ownerErr) + } + + db, dbErr := s.requireDB() + if dbErr != nil { + return dbErr + } + + row := db.QueryRowContext( + ctx, + `SELECT owner_pid, owner_identity_time FROM resource_locks WHERE resource_key = ?`, + resourceKey, + ) + + var ownerPID int64 + var ownerIdentityTime string + scanErr := row.Scan(&ownerPID, &ownerIdentityTime) + if errors.Is(scanErr, sql.ErrNoRows) { + return fmt.Errorf("%w: %s", ErrResourceLeaseNotHeld, resourceKey) + } + if scanErr != nil { + return fmt.Errorf("could not verify resource lease '%s': %w", resourceKey, scanErr) + } + + heldOwner, heldOwnerErr := resourceLeaseOwnerFromDB(ownerPID, ownerIdentityTime) + if heldOwnerErr != nil { + return heldOwnerErr + } + if heldOwner.Pid != normalizedOwner.Pid || !heldOwner.IdentityTime.Equal(normalizedOwner.IdentityTime) { + return fmt.Errorf("%w: %s", ErrResourceLeaseNotHeld, resourceKey) + } + + return nil +} + +func (s *Store) WithResourceLease(ctx context.Context, resource LeasableResource, ownerProcess process.ProcessTreeItem, revalidationInterval time.Duration, f func(context.Context, *ResourceLease) error) error { + if f == nil { + return fmt.Errorf("%w: lease callback cannot be nil", ErrInvalidArgument) + } + + lease, acquireErr := s.AcquireResourceLease(ctx, resource, ownerProcess, revalidationInterval) + if acquireErr != nil { + return acquireErr + } + + callbackErr := f(ctx, lease) + releaseErr := s.ReleaseResourceLease(context.Background(), resource, ownerProcess) + return errors.Join(callbackErr, releaseErr) +} + +type inactiveResourceLeaseCandidate struct { + resourceKey string + ownerPID int64 + ownerIdentityTime string + updatedAtUnixNano int64 +} + +func (s *Store) DeleteInactiveResourceLeases(ctx context.Context) error { + candidates, candidatesErr := s.inactiveResourceLeaseCandidates(ctx) + if candidatesErr != nil { + return candidatesErr + } + if len(candidates) == 0 { + return nil + } + + return s.withImmediateTx(ctx, func(conn *sql.Conn) error { + for _, candidate := range candidates { + _, execErr := conn.ExecContext( + ctx, + `DELETE FROM resource_locks + WHERE resource_key = ? AND owner_pid = ? AND owner_identity_time = ? + AND updated_at_unix_nano = ?`, + candidate.resourceKey, + candidate.ownerPID, + candidate.ownerIdentityTime, + candidate.updatedAtUnixNano, + ) + if execErr != nil { + return fmt.Errorf("could not delete inactive resource lease '%s': %w", candidate.resourceKey, execErr) + } + } + return nil + }) +} + +func (s *Store) inactiveResourceLeaseCandidates(ctx context.Context) ([]inactiveResourceLeaseCandidate, error) { + db, dbErr := s.requireDB() + if dbErr != nil { + return nil, dbErr + } + + rows, queryErr := db.QueryContext( + ctx, + `SELECT resource_key, owner_pid, owner_identity_time, updated_at_unix_nano + FROM resource_locks`, + ) + if queryErr != nil { + return nil, fmt.Errorf("could not list resource leases for cleanup: %w", queryErr) + } + defer func() { + _ = rows.Close() + }() + + candidates := []inactiveResourceLeaseCandidate{} + for rows.Next() { + var candidate inactiveResourceLeaseCandidate + scanErr := rows.Scan( + &candidate.resourceKey, + &candidate.ownerPID, + &candidate.ownerIdentityTime, + &candidate.updatedAtUnixNano, + ) + if scanErr != nil { + return nil, fmt.Errorf("could not read resource lease for cleanup: %w", scanErr) + } + + if resourceLeaseIsInactive(candidate) { + candidates = append(candidates, candidate) + } + } + if rowsErr := rows.Err(); rowsErr != nil { + return nil, fmt.Errorf("could not list resource leases for cleanup: %w", rowsErr) + } + + return candidates, nil +} + +func resourceLeaseIsInactive(candidate inactiveResourceLeaseCandidate) bool { + ownerProcess, ownerErr := resourceLeaseOwnerFromDB(candidate.ownerPID, candidate.ownerIdentityTime) + return ownerErr != nil || !resourceLeaseOwnerIsActive(ownerProcess) +} + +func getResourceLease(ctx context.Context, conn *sql.Conn, resourceKey string) (*ResourceLease, error) { + row := conn.QueryRowContext( + ctx, + `SELECT resource_key, owner_pid, owner_identity_time, updated_at_unix_nano + FROM resource_locks WHERE resource_key = ?`, + resourceKey, + ) + + var lease ResourceLease + var ownerPID int64 + var ownerIdentityTime string + var updatedAtUnixNano int64 + scanErr := row.Scan( + &lease.ResourceKey, + &ownerPID, + &ownerIdentityTime, + &updatedAtUnixNano, + ) + if scanErr != nil { + return nil, fmt.Errorf("could not read resource lease '%s': %w", resourceKey, scanErr) + } + + ownerProcess, ownerErr := resourceLeaseOwnerFromDB(ownerPID, ownerIdentityTime) + if ownerErr != nil { + return nil, fmt.Errorf("could not read resource lease owner '%s': %w", resourceKey, ownerErr) + } + lease.OwnerProcess = ownerProcess + lease.UpdatedAt = timeFromUnixNano(updatedAtUnixNano) + return &lease, nil +} + +func updateResourceLeaseTimestamp(ctx context.Context, conn *sql.Conn, resourceKey string, lease *ResourceLease, now time.Time) error { + _, execErr := conn.ExecContext( + ctx, + `UPDATE resource_locks + SET updated_at_unix_nano = ? + WHERE resource_key = ? AND owner_pid = ? AND owner_identity_time = ? AND updated_at_unix_nano = ?`, + unixNano(now), + resourceKey, + lease.OwnerProcess.Pid, + timeString(lease.OwnerProcess.IdentityTime), + unixNano(lease.UpdatedAt), + ) + if execErr != nil { + return fmt.Errorf("could not update resource lease validation time '%s': %w", resourceKey, execErr) + } + return nil +} + +func updateResourceLeaseOwner(ctx context.Context, conn *sql.Conn, resourceKey string, ownerProcess process.ProcessTreeItem, now time.Time) error { + _, execErr := conn.ExecContext( + ctx, + `UPDATE resource_locks + SET owner_pid = ?, owner_identity_time = ?, updated_at_unix_nano = ? + WHERE resource_key = ?`, + ownerProcess.Pid, + timeString(ownerProcess.IdentityTime), + unixNano(now), + resourceKey, + ) + if execErr != nil { + return fmt.Errorf("could not update resource lease owner '%s': %w", resourceKey, execErr) + } + return nil +} diff --git a/internal/statestore/migrations/000001_initial.up.sql b/internal/statestore/migrations/000001_initial.up.sql new file mode 100644 index 00000000..5b878359 --- /dev/null +++ b/internal/statestore/migrations/000001_initial.up.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS resource_locks ( + resource_key TEXT PRIMARY KEY, + owner_pid INTEGER NOT NULL, + owner_identity_time TEXT NOT NULL, + updated_at_unix_nano INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS persistent_processes ( + resource_key TEXT PRIMARY KEY, + lifecycle_key TEXT NOT NULL, + pid INTEGER NOT NULL, + identity_time TEXT NOT NULL, + run_id TEXT NOT NULL, + stdout_file TEXT NOT NULL, + stderr_file TEXT NOT NULL, + lifecycle_metadata TEXT NOT NULL DEFAULT '', + updated_at_unix_nano INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_resource_locks_updated_at ON resource_locks(updated_at_unix_nano); diff --git a/internal/statestore/migrations/README.md b/internal/statestore/migrations/README.md new file mode 100644 index 00000000..c5c269f0 --- /dev/null +++ b/internal/statestore/migrations/README.md @@ -0,0 +1,20 @@ +# State store migration authoring rules + +DCP applies these SQLite migrations with `golang-migrate/migrate` using the pure-Go `modernc.org/sqlite` driver. Migration files use golang-migrate names such as `000001_initial.up.sql`. DCP only applies `up` migrations at runtime; do not add destructive `down` migrations for the local state store. + +Migration SQL files are embedded into the DCP binary from `schema.go` with `go:embed`, so runtime schema initialization does not read migrations from the filesystem. + +The local state store is shared by multiple DCP processes and may be opened by older DCP binaries after a newer binary has run. Do not treat the migration version as a global compatibility gate. Runtime code that needs a newer schema shape should probe the specific table, column, or capability it needs and fail only that feature with an actionable error. + +## Compatibility rules + +- Prefer expand/migrate/contract. Add compatible schema first, teach code to dual-read or dual-write where needed, and only remove old schema after the supported compatibility window has intentionally ended. +- Application SQL must use explicit column lists for inserts and selects so added columns do not break older binaries. +- Allowed by default: `CREATE TABLE IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, and `ALTER TABLE ... ADD COLUMN` when the column is nullable or has a safe constant `DEFAULT`. +- Allowed with care: non-unique indexes, idempotent backfills that preserve existing column meaning, and additive columns that newer code can ignore when absent. +- Requires explicit compatibility design: unique indexes, check constraints, foreign keys, changed primary keys, table rebuilds, or any constraint that can reject writes older DCP binaries still perform. +- Do not add a `NOT NULL` column without a default to an existing table. +- Do not rename or drop tables/columns, change column meaning, or make older binaries blind to locks/process records during the compatibility window. +- Treat `resource_locks` as ephemeral but still compatibility-sensitive: old and new DCP instances must coordinate through the same lock representation until old writers are no longer supported. +- Treat `persistent_processes` as durable metadata. Migrations must preserve existing records unless an explicit product decision retires the stored state. +- Do not include `BEGIN` or `COMMIT` statements. The golang-migrate SQLite driver wraps each migration in a transaction. diff --git a/internal/statestore/processes.go b/internal/statestore/processes.go new file mode 100644 index 00000000..4b104bff --- /dev/null +++ b/internal/statestore/processes.go @@ -0,0 +1,205 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package statestore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/microsoft/dcp/pkg/process" +) + +var ErrPersistentProcessNotFound = errors.New("persistent process record not found") + +type PersistentProcessRecord struct { + ResourceKey string + LifecycleKey string + PID process.Pid_t + IdentityTime time.Time + RunID string + StdOutFile string + StdErrFile string + LifecycleMetadata string + UpdatedAt time.Time +} + +func (s *Store) UpsertPersistentProcess(ctx context.Context, record PersistentProcessRecord) error { + record.ResourceKey = strings.TrimSpace(record.ResourceKey) + if record.ResourceKey == "" { + return fmt.Errorf("%w: persistent process resource key cannot be empty", ErrInvalidArgument) + } + if record.PID <= 0 { + return fmt.Errorf("%w: persistent process PID must be positive", ErrInvalidArgument) + } + if record.IdentityTime.IsZero() { + return fmt.Errorf("%w: persistent process identity time cannot be zero", ErrInvalidArgument) + } + if strings.TrimSpace(record.RunID) == "" { + return fmt.Errorf("%w: persistent process run ID cannot be empty", ErrInvalidArgument) + } + + now := time.Now().UTC() + return s.withImmediateTx(ctx, func(conn *sql.Conn) error { + _, execErr := conn.ExecContext( + ctx, + `INSERT INTO persistent_processes( + resource_key, lifecycle_key, pid, + identity_time, run_id, + stdout_file, stderr_file, lifecycle_metadata, updated_at_unix_nano + ) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(resource_key) DO UPDATE SET + lifecycle_key = excluded.lifecycle_key, + pid = excluded.pid, + identity_time = excluded.identity_time, + run_id = excluded.run_id, + stdout_file = excluded.stdout_file, + stderr_file = excluded.stderr_file, + lifecycle_metadata = excluded.lifecycle_metadata, + updated_at_unix_nano = excluded.updated_at_unix_nano`, + record.ResourceKey, + record.LifecycleKey, + int64(record.PID), + timeString(record.IdentityTime), + record.RunID, + record.StdOutFile, + record.StdErrFile, + record.LifecycleMetadata, + unixNano(now), + ) + if execErr != nil { + return fmt.Errorf("could not upsert persistent process record '%s': %w", record.ResourceKey, execErr) + } + + return nil + }) +} + +func (s *Store) GetPersistentProcess(ctx context.Context, resourceKey string) (*PersistentProcessRecord, error) { + resourceKey = strings.TrimSpace(resourceKey) + if resourceKey == "" { + return nil, fmt.Errorf("%w: persistent process resource key cannot be empty", ErrInvalidArgument) + } + + db, dbErr := s.requireDB() + if dbErr != nil { + return nil, dbErr + } + + row := db.QueryRowContext( + ctx, + `SELECT resource_key, lifecycle_key, pid, + identity_time, run_id, + stdout_file, stderr_file, lifecycle_metadata, updated_at_unix_nano + FROM persistent_processes WHERE resource_key = ?`, + resourceKey, + ) + + record, scanErr := scanPersistentProcess(row) + if errors.Is(scanErr, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrPersistentProcessNotFound, resourceKey) + } + if scanErr != nil { + return nil, fmt.Errorf("could not get persistent process record '%s': %w", resourceKey, scanErr) + } + + return record, nil +} + +func (s *Store) ListPersistentProcesses(ctx context.Context) ([]PersistentProcessRecord, error) { + db, dbErr := s.requireDB() + if dbErr != nil { + return nil, dbErr + } + + rows, queryErr := db.QueryContext( + ctx, + `SELECT resource_key, lifecycle_key, pid, + identity_time, run_id, + stdout_file, stderr_file, lifecycle_metadata, updated_at_unix_nano + FROM persistent_processes + ORDER BY resource_key`, + ) + if queryErr != nil { + return nil, fmt.Errorf("could not list persistent process records: %w", queryErr) + } + defer func() { + _ = rows.Close() + }() + + records := []PersistentProcessRecord{} + for rows.Next() { + record, scanErr := scanPersistentProcess(rows) + if scanErr != nil { + return nil, fmt.Errorf("could not read persistent process record: %w", scanErr) + } + records = append(records, *record) + } + if rowsErr := rows.Err(); rowsErr != nil { + return nil, fmt.Errorf("could not list persistent process records: %w", rowsErr) + } + + return records, nil +} + +func (s *Store) DeletePersistentProcess(ctx context.Context, resourceKey string) error { + resourceKey = strings.TrimSpace(resourceKey) + if resourceKey == "" { + return fmt.Errorf("%w: persistent process resource key cannot be empty", ErrInvalidArgument) + } + + return s.withImmediateTx(ctx, func(conn *sql.Conn) error { + _, execErr := conn.ExecContext(ctx, `DELETE FROM persistent_processes WHERE resource_key = ?`, resourceKey) + if execErr != nil { + return fmt.Errorf("could not delete persistent process record '%s': %w", resourceKey, execErr) + } + return nil + }) +} + +type persistentProcessScanner interface { + Scan(dest ...any) error +} + +func scanPersistentProcess(row persistentProcessScanner) (*PersistentProcessRecord, error) { + var record PersistentProcessRecord + var pid int64 + var identityTime string + var updatedAtUnixNano int64 + + scanErr := row.Scan( + &record.ResourceKey, + &record.LifecycleKey, + &pid, + &identityTime, + &record.RunID, + &record.StdOutFile, + &record.StdErrFile, + &record.LifecycleMetadata, + &updatedAtUnixNano, + ) + if scanErr != nil { + return nil, scanErr + } + + pidValue, pidErr := process.Int64_ToPidT(pid) + if pidErr != nil { + return nil, pidErr + } + + record.PID = pidValue + identityTimeValue, identityTimeErr := timeFromString(identityTime) + if identityTimeErr != nil { + return nil, identityTimeErr + } + record.IdentityTime = identityTimeValue + record.UpdatedAt = timeFromUnixNano(updatedAtUnixNano) + return &record, nil +} diff --git a/internal/statestore/schema.go b/internal/statestore/schema.go new file mode 100644 index 00000000..777f1fba --- /dev/null +++ b/internal/statestore/schema.go @@ -0,0 +1,169 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package statestore + +import ( + "context" + "database/sql" + "embed" + "errors" + "fmt" + "time" + + gomigrate "github.com/golang-migrate/migrate/v4" + migratesqlite "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source/iofs" + + "github.com/microsoft/dcp/internal/lockfile" +) + +const ( + currentSchemaVersion = 1 + + migrationTableName = "schema_migrations" + migrationLockFileSuffix = ".migrate.lock" +) + +// migrationFiles embeds the SQL migration files into the DCP binary so runtime +// schema initialization does not depend on external files being present. +// +//go:embed migrations/*.sql +var migrationFiles embed.FS + +func (s *Store) migrate(ctx context.Context, migrationDB *sql.DB, busyTimeout time.Duration) (err error) { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if _, dbErr := s.requireDB(); dbErr != nil { + return dbErr + } + if migrationDB == nil { + return fmt.Errorf("%w: migration database is nil", ErrInvalidArgument) + } + + migrationLock, lockErr := lockfile.NewLockfile(s.path + migrationLockFileSuffix) + if lockErr != nil { + return fmt.Errorf("could not create state store migration lock: %w", lockErr) + } + if lockErr = migrationLock.TryLock(ctx, lockfile.DefaultLockRetryInterval); lockErr != nil { + closeErr := migrationLock.Close() + return fmt.Errorf("could not acquire state store migration lock: %w", errors.Join(lockErr, closeErr)) + } + defer func() { + err = errors.Join(err, migrationLock.Close()) + }() + + if migrationErr := s.dropPreMigrationLegacyResourceLocksTable(ctx); migrationErr != nil { + return migrationErr + } + + return s.runMigrations(ctx, migrationDB, busyTimeout) +} + +func (s *Store) runMigrations(ctx context.Context, migrationDB *sql.DB, busyTimeout time.Duration) (err error) { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + + if migrationDB == nil { + return fmt.Errorf("%w: migration database is nil", ErrInvalidArgument) + } + + sourceDriver, sourceErr := iofs.New(migrationFiles, "migrations") + if sourceErr != nil { + return fmt.Errorf("could not load state store migrations: %w", sourceErr) + } + + databaseDriver, databaseDriverErr := migratesqlite.WithInstance(migrationDB, &migratesqlite.Config{ + DatabaseName: s.path, + MigrationsTable: migrationTableName, + }) + if databaseDriverErr != nil { + sourceCloseErr := sourceDriver.Close() + return fmt.Errorf("could not initialize state store migration database driver: %w", errors.Join(databaseDriverErr, sourceCloseErr)) + } + + migrationRunner, runnerErr := gomigrate.NewWithInstance("iofs", sourceDriver, sqliteDriverName, databaseDriver) + if runnerErr != nil { + sourceCloseErr := sourceDriver.Close() + databaseCloseErr := databaseDriver.Close() + return fmt.Errorf("could not initialize state store migration runner: %w", errors.Join(runnerErr, sourceCloseErr, databaseCloseErr)) + } + defer func() { + sourceCloseErr, databaseCloseErr := migrationRunner.Close() + err = errors.Join(err, sourceCloseErr, databaseCloseErr) + }() + migrationRunner.LockTimeout = busyTimeout + + if migrationErr := migrationRunner.Up(); migrationErr != nil && !errors.Is(migrationErr, gomigrate.ErrNoChange) { + return fmt.Errorf("could not apply state store migrations: %w", migrationErr) + } + + return nil +} + +func (s *Store) dropPreMigrationLegacyResourceLocksTable(ctx context.Context) error { + // Early development builds created an unversioned resource_locks table with + // string owners. Future schema changes should be expressed as numbered + // migrations instead of adding more pre-migration repair paths. + return s.withImmediateTx(ctx, func(conn *sql.Conn) error { + return dropLegacyResourceLocksTable(ctx, conn) + }) +} + +func dropLegacyResourceLocksTable(ctx context.Context, conn *sql.Conn) error { + hasLegacyOwnerColumn, legacyOwnerColumnErr := resourceLocksHasColumn(ctx, conn, "owner_instance_id") + if legacyOwnerColumnErr != nil { + return legacyOwnerColumnErr + } + if !hasLegacyOwnerColumn { + return nil + } + + hasOwnerPIDColumn, ownerPIDColumnErr := resourceLocksHasColumn(ctx, conn, "owner_pid") + if ownerPIDColumnErr != nil { + return ownerPIDColumnErr + } + if hasOwnerPIDColumn { + return nil + } + + if _, dropErr := conn.ExecContext(ctx, `DROP TABLE resource_locks`); dropErr != nil { + return fmt.Errorf("could not drop legacy resource_locks table: %w", dropErr) + } + return nil +} + +func resourceLocksHasColumn(ctx context.Context, conn *sql.Conn, columnName string) (bool, error) { + rows, queryErr := conn.QueryContext(ctx, `PRAGMA table_info(resource_locks)`) + if queryErr != nil { + return false, fmt.Errorf("could not inspect resource_locks schema: %w", queryErr) + } + defer func() { + _ = rows.Close() + }() + + for rows.Next() { + var cid int + var name string + var typeName string + var notNull int + var defaultValue sql.NullString + var primaryKey int + scanErr := rows.Scan(&cid, &name, &typeName, ¬Null, &defaultValue, &primaryKey) + if scanErr != nil { + return false, fmt.Errorf("could not read resource_locks schema: %w", scanErr) + } + if name == columnName { + return true, nil + } + } + if rowsErr := rows.Err(); rowsErr != nil { + return false, fmt.Errorf("could not inspect resource_locks schema: %w", rowsErr) + } + + return false, nil +} diff --git a/internal/statestore/store.go b/internal/statestore/store.go new file mode 100644 index 00000000..99e3c9f5 --- /dev/null +++ b/internal/statestore/store.go @@ -0,0 +1,366 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Package statestore provides the local, SQLite-backed DCP state store. +package statestore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + _ "modernc.org/sqlite" + + "github.com/microsoft/dcp/internal/dcppaths" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" +) + +const ( + DCP_STATE_STORE_PATH = "DCP_STATE_STORE_PATH" + + sqliteDriverName = "sqlite" + + defaultStoreDirName = "state" + defaultElevatedStoreDirName = "state.elevated" + defaultStoreFileName = "state.sqlite3" + + DefaultBusyTimeout = 5 * time.Second + + // DCP's state store does small, frequent writes with concurrent readers. A 256-page threshold + // keeps the WAL bounded to about 1 MiB with SQLite's default 4 KiB page size without + // checkpointing on every small transaction. + defaultWALAutoCheckpointPages = 256 +) + +var ( + ErrStoreNotInitialized = errors.New("state store is not initialized") + ErrInvalidArgument = errors.New("invalid state store argument") +) + +type Options struct { + // Path is the SQLite database file path. If empty, the default DCP user store path is used. + Path string + + // BusyTimeout is the SQLite busy timeout used when another process holds a database lock. + BusyTimeout time.Duration +} + +type Store struct { + db *sql.DB + path string +} + +func DefaultPath() (string, error) { + stateStorePath, _, stateStorePathErr := resolveStateStorePath("") + return stateStorePath, stateStorePathErr +} + +func resolveStateStorePath(path string) (string, bool, error) { + if storePath := strings.TrimSpace(path); storePath != "" { + return storePath, true, nil + } + + if stateStorePath, found := os.LookupEnv(DCP_STATE_STORE_PATH); found && strings.TrimSpace(stateStorePath) != "" { + return strings.TrimSpace(stateStorePath), true, nil + } + + dcpFolder, dcpFolderErr := dcppaths.EnsureUserDcpDir() + if dcpFolderErr != nil { + return "", false, dcpFolderErr + } + + isAdmin, isAdminErr := osutil.IsAdmin() + if isAdminErr != nil { + return "", false, isAdminErr + } + + return defaultStateStorePath(dcpFolder, isAdmin), false, nil +} + +func defaultStateStorePath(dcpFolder string, isAdmin bool) string { + if isAdmin { + return filepath.Join(dcpFolder, defaultElevatedStoreDirName, defaultStoreFileName) + } + return filepath.Join(dcpFolder, defaultStoreDirName, defaultStoreFileName) +} + +func Open(ctx context.Context, options Options) (*Store, error) { + storePath, explicitPath, stateStorePathErr := resolveStateStorePath(options.Path) + if stateStorePathErr != nil { + return nil, fmt.Errorf("could not determine state store path: %w", stateStorePathErr) + } + + absPath, absPathErr := filepath.Abs(storePath) + if absPathErr != nil { + return nil, fmt.Errorf("could not determine absolute state store path for '%s': %w", storePath, absPathErr) + } + + if ensureErr := ensureStateStoreDir(absPath, explicitPath); ensureErr != nil { + return nil, ensureErr + } + + busyTimeout := options.BusyTimeout + if busyTimeout <= 0 { + busyTimeout = DefaultBusyTimeout + } + + db, openErr := openSQLiteDB(ctx, absPath, busyTimeout) + if openErr != nil { + return nil, fmt.Errorf("could not initialize state store database '%s': %w", absPath, openErr) + } + + store := &Store{ + db: db, + path: absPath, + } + + if configureErr := store.configure(ctx, busyTimeout); configureErr != nil { + closeErr := db.Close() + return nil, fmt.Errorf("could not configure state store database '%s': %w", absPath, errors.Join(configureErr, closeErr)) + } + + migrationDB, migrationOpenErr := openSQLiteDB(ctx, absPath, busyTimeout) + if migrationOpenErr != nil { + closeErr := db.Close() + return nil, fmt.Errorf("could not initialize state store migration database '%s': %w", absPath, errors.Join(migrationOpenErr, closeErr)) + } + if migrateErr := store.migrate(ctx, migrationDB, busyTimeout); migrateErr != nil { + migrationCloseErr := migrationDB.Close() + closeErr := db.Close() + return nil, fmt.Errorf("could not migrate state store database '%s': %w", absPath, errors.Join(migrateErr, migrationCloseErr, closeErr)) + } + if migrationCloseErr := migrationDB.Close(); migrationCloseErr != nil { + closeErr := db.Close() + return nil, fmt.Errorf("could not close state store migration database '%s': %w", absPath, errors.Join(migrationCloseErr, closeErr)) + } + + return store, nil +} + +func openSQLiteDB(ctx context.Context, path string, busyTimeout time.Duration) (*sql.DB, error) { + db, openErr := sql.Open(sqliteDriverName, sqliteDSN(path, busyTimeout)) + if openErr != nil { + return nil, openErr + } + + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + db.SetConnMaxLifetime(0) + + if pingErr := db.PingContext(ctx); pingErr != nil { + closeErr := db.Close() + return nil, errors.Join(pingErr, closeErr) + } + + return db, nil +} + +func sqliteDSN(path string, busyTimeout time.Duration) string { + sqlitePath := strings.ReplaceAll(path, "\\", "/") + dsn := url.URL{Scheme: "file"} + if isWindowsDrivePath(sqlitePath) { + dsn.Opaque = sqlitePath + } else { + dsn.Path = sqlitePath + } + query := dsn.Query() + query.Add("_pragma", fmt.Sprintf("busy_timeout=%d", busyTimeout.Milliseconds())) + dsn.RawQuery = query.Encode() + return dsn.String() +} + +func isWindowsDrivePath(path string) bool { + if len(path) < 3 || path[1] != ':' || path[2] != '/' { + return false + } + driveLetter := path[0] + return (driveLetter >= 'A' && driveLetter <= 'Z') || (driveLetter >= 'a' && driveLetter <= 'z') +} + +func (s *Store) Path() string { + if s == nil { + return "" + } + return s.path +} + +func (s *Store) Close() error { + if s == nil || s.db == nil { + return nil + } + + closeErr := s.db.Close() + s.db = nil + return closeErr +} + +func (s *Store) requireDB() (*sql.DB, error) { + if s == nil || s.db == nil { + return nil, ErrStoreNotInitialized + } + return s.db, nil +} + +func (s *Store) configure(ctx context.Context, busyTimeout time.Duration) error { + db, dbErr := s.requireDB() + if dbErr != nil { + return dbErr + } + + busyTimeoutMilliseconds := busyTimeout.Milliseconds() + if busyTimeoutMilliseconds <= 0 { + busyTimeoutMilliseconds = 1 + } + + if _, busyErr := db.ExecContext(ctx, fmt.Sprintf("PRAGMA busy_timeout = %d", busyTimeoutMilliseconds)); busyErr != nil { + return fmt.Errorf("could not configure SQLite busy timeout: %w", busyErr) + } + + var journalMode string + journalRow := db.QueryRowContext(ctx, "PRAGMA journal_mode = WAL") + if journalErr := journalRow.Scan(&journalMode); journalErr != nil { + return fmt.Errorf("could not enable SQLite WAL mode: %w", journalErr) + } + if !strings.EqualFold(journalMode, "wal") { + return fmt.Errorf("could not enable SQLite WAL mode: got journal mode %q", journalMode) + } + + if _, syncErr := db.ExecContext(ctx, "PRAGMA synchronous = NORMAL"); syncErr != nil { + return fmt.Errorf("could not configure SQLite synchronous mode: %w", syncErr) + } + + var autoCheckpointPages int + autoCheckpointRow := db.QueryRowContext(ctx, fmt.Sprintf("PRAGMA wal_autocheckpoint = %d", defaultWALAutoCheckpointPages)) + if autoCheckpointErr := autoCheckpointRow.Scan(&autoCheckpointPages); autoCheckpointErr != nil { + return fmt.Errorf("could not configure SQLite WAL auto-checkpoint threshold: %w", autoCheckpointErr) + } + if autoCheckpointPages != defaultWALAutoCheckpointPages { + return fmt.Errorf("could not configure SQLite WAL auto-checkpoint threshold: got %d pages", autoCheckpointPages) + } + + if _, foreignKeyErr := db.ExecContext(ctx, "PRAGMA foreign_keys = ON"); foreignKeyErr != nil { + return fmt.Errorf("could not enable SQLite foreign key checks: %w", foreignKeyErr) + } + + return nil +} + +func ensureStateStoreDir(path string, explicitPath bool) error { + parentDir := filepath.Dir(path) + if explicitPath { + if ensureErr := ensureExplicitStateStoreDir(parentDir); ensureErr != nil { + return fmt.Errorf("could not prepare state store directory '%s': %w", parentDir, ensureErr) + } + return nil + } + if ensureErr := usvc_io.EnsureRestrictedDirectory(parentDir, osutil.PermissionOnlyOwnerReadWriteTraverse); ensureErr != nil { + return fmt.Errorf("could not prepare state store directory '%s': %w", parentDir, ensureErr) + } + return nil +} + +func ensureExplicitStateStoreDir(parentDir string) error { + createdDir := false + _, statErr := os.Lstat(parentDir) + if errors.Is(statErr, os.ErrNotExist) { + mkdirErr := os.Mkdir(parentDir, osutil.PermissionOnlyOwnerReadWriteTraverse) + if mkdirErr == nil { + createdDir = true + } else if !errors.Is(mkdirErr, os.ErrExist) { + return fmt.Errorf("could not create explicit state store directory: %w", mkdirErr) + } + } else if statErr != nil { + return fmt.Errorf("could not inspect explicit state store directory: %w", statErr) + } + + if createdDir { + if restrictErr := usvc_io.EnsureRestrictedDirectory(parentDir, osutil.PermissionOnlyOwnerReadWriteTraverse); restrictErr != nil { + return fmt.Errorf("could not restrict created explicit state store directory: %w", restrictErr) + } + return nil + } + + if validateErr := usvc_io.ValidateRestrictedDirectory(parentDir, osutil.PermissionOnlyOwnerReadWriteTraverse); validateErr != nil { + return fmt.Errorf("explicit state store directory must already be restricted: %w", validateErr) + } + return nil +} + +func (s *Store) withImmediateTx(ctx context.Context, f func(*sql.Conn) error) (err error) { + db, dbErr := s.requireDB() + if dbErr != nil { + return dbErr + } + + conn, connErr := db.Conn(ctx) + if connErr != nil { + return fmt.Errorf("could not get state store connection: %w", connErr) + } + + begun := false + committed := false + defer func() { + if begun && !committed { + _, rollbackErr := conn.ExecContext(context.Background(), "ROLLBACK") + err = errors.Join(err, rollbackErr) + } + closeErr := conn.Close() + err = errors.Join(err, closeErr) + }() + + if _, beginErr := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); beginErr != nil { + return fmt.Errorf("could not start state store transaction: %w", beginErr) + } + begun = true + + if fErr := f(conn); fErr != nil { + return fErr + } + + if _, commitErr := conn.ExecContext(ctx, "COMMIT"); commitErr != nil { + return fmt.Errorf("could not commit state store transaction: %w", commitErr) + } + committed = true + return nil +} + +func unixNano(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.UTC().UnixNano() +} + +func timeFromUnixNano(v int64) time.Time { + if v == 0 { + return time.Time{} + } + return time.Unix(0, v).UTC() +} + +func timeString(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} + +func timeFromString(v string) (time.Time, error) { + if v == "" { + return time.Time{}, nil + } + parsed, parseErr := time.Parse(time.RFC3339Nano, v) + if parseErr != nil { + return time.Time{}, parseErr + } + return parsed.UTC(), nil +} diff --git a/internal/statestore/store_test.go b/internal/statestore/store_test.go new file mode 100644 index 00000000..38228de7 --- /dev/null +++ b/internal/statestore/store_test.go @@ -0,0 +1,585 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package statestore + +import ( + "context" + "database/sql" + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/process" + "github.com/microsoft/dcp/pkg/testutil" +) + +const stateStoreTestTimeout = 10 * time.Second + +func openRawSQLiteDB(t *testing.T, ctx context.Context, path string) *sql.DB { + t.Helper() + + db, openErr := sql.Open(sqliteDriverName, sqliteDSN(path, 500*time.Millisecond)) + require.NoError(t, openErr) + + require.NoError(t, db.PingContext(ctx)) + return db +} + +func openTestStore(t *testing.T, ctx context.Context, path string) *Store { + t.Helper() + + require.NoError(t, usvc_io.EnsureRestrictedDirectory(filepath.Dir(path), osutil.PermissionOnlyOwnerReadWriteTraverse)) + store, openErr := Open(ctx, Options{ + Path: path, + BusyTimeout: 500 * time.Millisecond, + }) + require.NoError(t, openErr) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + return store +} + +type testLeasableResource string + +func (r testLeasableResource) GetLeaseKey() string { + return string(r) +} + +func TestSQLiteDSNUsesURIPathSeparators(t *testing.T) { + t.Parallel() + + require.Equal( + t, + "file:///tmp/dcp/state.sqlite3?_pragma=busy_timeout%3D500", + sqliteDSN("/tmp/dcp/state.sqlite3", 500*time.Millisecond), + ) + require.Equal( + t, + "file:C:/Users/runner/AppData/Local/Temp/state.sqlite3?_pragma=busy_timeout%3D500", + sqliteDSN(`C:\Users\runner\AppData\Local\Temp\state.sqlite3`, 500*time.Millisecond), + ) +} + +func TestDefaultStateStoreDirRestrictsLeafDirectoryOnly(t *testing.T) { + t.Parallel() + + dcpFolder := filepath.Join(t.TempDir(), ".dcp") + require.NoError(t, os.Mkdir(dcpFolder, osutil.PermissionDirectoryOthersRead)) + + for _, isAdmin := range []bool{false, true} { + stateStorePath := defaultStateStorePath(dcpFolder, isAdmin) + require.NoError(t, ensureStateStoreDir(stateStorePath, false)) + + stateStoreDir := filepath.Dir(stateStorePath) + require.DirExists(t, stateStoreDir) + require.NoError(t, usvc_io.ValidateRestrictedDirectory(stateStoreDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) + } + if runtime.GOOS != "windows" { + rootInfo, rootStatErr := os.Lstat(dcpFolder) + require.NoError(t, rootStatErr) + require.Equal(t, osutil.PermissionDirectoryOthersRead, rootInfo.Mode().Perm()) + } +} + +func requireMigrationVersion(t *testing.T, ctx context.Context, store *Store, expectedVersion int) { + t.Helper() + + row := store.db.QueryRowContext(ctx, `SELECT version, dirty FROM schema_migrations LIMIT 1`) + var version int + var dirty bool + require.NoError(t, row.Scan(&version, &dirty)) + require.Equal(t, expectedVersion, version) + require.False(t, dirty) +} + +func createUnversionedCurrentSchema(t *testing.T, ctx context.Context, path string) { + t.Helper() + + db := openRawSQLiteDB(t, ctx, path) + defer func() { + require.NoError(t, db.Close()) + }() + + initialMigration, readErr := fs.ReadFile(migrationFiles, "migrations/000001_initial.up.sql") + require.NoError(t, readErr) + + _, execErr := db.ExecContext(ctx, string(initialMigration)) + require.NoError(t, execErr) +} + +func testResourceLeaseOwner(t *testing.T, identityOffset time.Duration) (process.ProcessTreeItem, error) { + t.Helper() + + currentProcess, currentProcessErr := process.This() + if currentProcessErr != nil { + return process.ProcessTreeItem{}, currentProcessErr + } + + currentProcess.IdentityTime = currentProcess.IdentityTime.Add(identityOffset) + return normalizeResourceLeaseOwner(currentProcess) +} + +func setResourceLeaseUpdatedAt(t *testing.T, ctx context.Context, store *Store, resourceKey string, updatedAt time.Time) { + t.Helper() + + _, updateErr := store.db.ExecContext( + ctx, + `UPDATE resource_locks SET updated_at_unix_nano = ? WHERE resource_key = ?`, + unixNano(updatedAt), + resourceKey, + ) + require.NoError(t, updateErr) +} + +func TestOpenCreatesSchema(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store := openTestStore(t, ctx, storePath) + + require.Equal(t, storePath, store.Path()) + require.FileExists(t, storePath) + + requireMigrationVersion(t, ctx, store, currentSchemaVersion) +} + +func TestOpenWithExplicitPathRejectsPermissiveExistingParentDirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows directory permissions are not represented by Unix mode bits") + } + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + + stateStoreDir := filepath.Join(t.TempDir(), "state-store") + require.NoError(t, os.Mkdir(stateStoreDir, osutil.PermissionDirectoryOthersRead)) + require.NoError(t, os.Chmod(stateStoreDir, osutil.PermissionDirectoryOthersRead)) + storePath := filepath.Join(stateStoreDir, "state.sqlite3") + + _, openErr := Open(ctx, Options{Path: storePath}) + + require.ErrorContains(t, openErr, "explicit state store directory must already be restricted") + info, statErr := os.Lstat(stateStoreDir) + require.NoError(t, statErr) + require.Equal(t, osutil.PermissionDirectoryOthersRead, info.Mode().Perm()) +} + +func TestOpenWithEnvPathRejectsPermissiveExistingParentDirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows directory permissions are not represented by Unix mode bits") + } + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + + stateStoreDir := filepath.Join(t.TempDir(), "state-store") + require.NoError(t, os.Mkdir(stateStoreDir, osutil.PermissionDirectoryOthersRead)) + require.NoError(t, os.Chmod(stateStoreDir, osutil.PermissionDirectoryOthersRead)) + storePath := filepath.Join(stateStoreDir, "state.sqlite3") + t.Setenv(DCP_STATE_STORE_PATH, storePath) + + _, openErr := Open(ctx, Options{}) + + require.ErrorContains(t, openErr, "explicit state store directory must already be restricted") + info, statErr := os.Lstat(stateStoreDir) + require.NoError(t, statErr) + require.Equal(t, osutil.PermissionDirectoryOthersRead, info.Mode().Perm()) +} + +func TestOpenWithExplicitPathCreatesMissingParentDirectoryRestricted(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + + stateStoreDir := filepath.Join(t.TempDir(), "state-store") + storePath := filepath.Join(stateStoreDir, "state.sqlite3") + + store, openErr := Open(ctx, Options{ + Path: storePath, + BusyTimeout: 500 * time.Millisecond, + }) + require.NoError(t, openErr) + t.Cleanup(func() { + require.NoError(t, store.Close()) + }) + + require.Equal(t, storePath, store.Path()) + require.FileExists(t, storePath) + info, statErr := os.Lstat(stateStoreDir) + require.NoError(t, statErr) + require.True(t, info.IsDir()) + require.NoError(t, usvc_io.ValidateRestrictedDirectory(stateStoreDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) + if runtime.GOOS != "windows" { + require.Equal(t, osutil.PermissionOnlyOwnerReadWriteTraverse, info.Mode().Perm()) + } +} + +func TestOpenConfiguresWALAutoCheckpoint(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store := openTestStore(t, ctx, storePath) + + row := store.db.QueryRowContext(ctx, "PRAGMA wal_autocheckpoint") + var autoCheckpointPages int + require.NoError(t, row.Scan(&autoCheckpointPages)) + require.Equal(t, defaultWALAutoCheckpointPages, autoCheckpointPages) +} + +func TestOpenMigratesUnversionedCurrentSchemaWithoutLosingResourceLocks(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + createUnversionedCurrentSchema(t, ctx, storePath) + + db := openRawSQLiteDB(t, ctx, storePath) + owner, ownerErr := testResourceLeaseOwner(t, 0) + require.NoError(t, ownerErr) + + now := time.Now().UTC() + _, insertErr := db.ExecContext( + ctx, + `INSERT INTO resource_locks(resource_key, owner_pid, owner_identity_time, updated_at_unix_nano) + VALUES(?, ?, ?, ?)`, + "container/existing", + owner.Pid, + timeString(owner.IdentityTime), + unixNano(now), + ) + require.NoError(t, insertErr) + require.NoError(t, db.Close()) + + store := openTestStore(t, ctx, storePath) + requireMigrationVersion(t, ctx, store, currentSchemaVersion) + + otherOwner, otherOwnerErr := testResourceLeaseOwner(t, -time.Hour) + require.NoError(t, otherOwnerErr) + _, acquireErr := store.AcquireResourceLease(ctx, testLeasableResource("container/existing"), otherOwner, time.Minute) + require.ErrorIs(t, acquireErr, ErrResourceLeaseHeld) +} + +func TestOpenMigratesUnversionedLegacyResourceLocksTable(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + + db := openRawSQLiteDB(t, ctx, storePath) + _, createErr := db.ExecContext( + ctx, + `CREATE TABLE resource_locks ( + resource_key TEXT PRIMARY KEY, + owner_instance_id TEXT NOT NULL, + updated_at_unix_nano INTEGER NOT NULL + )`, + ) + require.NoError(t, createErr) + require.NoError(t, db.Close()) + + store := openTestStore(t, ctx, storePath) + requireMigrationVersion(t, ctx, store, currentSchemaVersion) + + conn, connErr := store.db.Conn(ctx) + require.NoError(t, connErr) + defer func() { + require.NoError(t, conn.Close()) + }() + + hasOwnerPIDColumn, ownerPIDColumnErr := resourceLocksHasColumn(ctx, conn, "owner_pid") + require.NoError(t, ownerPIDColumnErr) + require.True(t, hasOwnerPIDColumn) + + hasLegacyOwnerColumn, legacyOwnerColumnErr := resourceLocksHasColumn(ctx, conn, "owner_instance_id") + require.NoError(t, legacyOwnerColumnErr) + require.False(t, hasLegacyOwnerColumn) +} + +func TestResourceLeaseCoordinatesAcrossStoreHandles(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store1 := openTestStore(t, ctx, storePath) + store2 := openTestStore(t, ctx, storePath) + + owner1, owner1Err := testResourceLeaseOwner(t, 0) + require.NoError(t, owner1Err) + owner2, owner2Err := testResourceLeaseOwner(t, -time.Hour) + require.NoError(t, owner2Err) + + lease, acquireErr := store1.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner1, time.Minute) + require.NoError(t, acquireErr) + require.Equal(t, owner1, lease.OwnerProcess) + + _, blockedErr := store2.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner2, time.Minute) + require.ErrorIs(t, blockedErr, ErrResourceLeaseHeld) + heldLease, foundHeldLease := HeldResourceLease(blockedErr) + require.True(t, foundHeldLease) + require.Equal(t, "container/test", heldLease.ResourceKey) + require.Equal(t, owner1, heldLease.OwnerProcess) + + require.NoError(t, store1.ReleaseResourceLease(ctx, testLeasableResource("container/test"), owner1)) + + lease, acquireErr = store2.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner2, time.Minute) + require.NoError(t, acquireErr) + require.Equal(t, owner2, lease.OwnerProcess) +} + +func TestResourceLeasePreservesOutOfUnixNanoRangeOwnerIdentityTime(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store := openTestStore(t, ctx, storePath) + + owner, ownerErr := normalizeResourceLeaseOwner(process.ProcessTreeItem{ + Pid: process.Pid_t(1234), + IdentityTime: time.Date(1, time.January, 1, 0, 3, 10, 720000000, time.UTC), + }) + require.NoError(t, ownerErr) + + lease, acquireErr := store.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner, time.Minute) + require.NoError(t, acquireErr) + require.Equal(t, owner, lease.OwnerProcess) + + require.NoError(t, store.ReleaseResourceLease(ctx, testLeasableResource("container/test"), owner)) +} + +func TestResourceLeaseReleaseRequiresOwnedLease(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store := openTestStore(t, ctx, storePath) + + owner1, owner1Err := testResourceLeaseOwner(t, 0) + require.NoError(t, owner1Err) + owner2, owner2Err := testResourceLeaseOwner(t, -time.Hour) + require.NoError(t, owner2Err) + + missingReleaseErr := store.ReleaseResourceLease(ctx, testLeasableResource("container/missing"), owner1) + require.ErrorIs(t, missingReleaseErr, ErrResourceLeaseNotHeld) + + _, acquireErr := store.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner1, time.Minute) + require.NoError(t, acquireErr) + + wrongOwnerReleaseErr := store.ReleaseResourceLease(ctx, testLeasableResource("container/test"), owner2) + require.ErrorIs(t, wrongOwnerReleaseErr, ErrResourceLeaseNotHeld) + + require.NoError(t, store.ReleaseResourceLease(ctx, testLeasableResource("container/test"), owner1)) +} + +func TestResourceLeaseVerifyRequiresOwnedLease(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store := openTestStore(t, ctx, storePath) + + owner1, owner1Err := testResourceLeaseOwner(t, 0) + require.NoError(t, owner1Err) + owner2, owner2Err := testResourceLeaseOwner(t, -time.Hour) + require.NoError(t, owner2Err) + + missingVerifyErr := store.VerifyResourceLeaseHeld(ctx, testLeasableResource("container/missing"), owner1) + require.ErrorIs(t, missingVerifyErr, ErrResourceLeaseNotHeld) + + _, acquireErr := store.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner1, time.Minute) + require.NoError(t, acquireErr) + + wrongOwnerVerifyErr := store.VerifyResourceLeaseHeld(ctx, testLeasableResource("container/test"), owner2) + require.ErrorIs(t, wrongOwnerVerifyErr, ErrResourceLeaseNotHeld) + + require.NoError(t, store.VerifyResourceLeaseHeld(ctx, testLeasableResource("container/test"), owner1)) +} + +func TestResourceLeaseDoesNotExpireWhileOwnerIsActive(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store1 := openTestStore(t, ctx, storePath) + store2 := openTestStore(t, ctx, storePath) + + owner1, owner1Err := testResourceLeaseOwner(t, 0) + require.NoError(t, owner1Err) + owner2, owner2Err := testResourceLeaseOwner(t, -time.Hour) + require.NoError(t, owner2Err) + + _, acquireErr := store1.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner1, time.Minute) + require.NoError(t, acquireErr) + setResourceLeaseUpdatedAt(t, ctx, store1, "container/test", time.Now().UTC().Add(-time.Hour)) + + _, retryErr := store2.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner2, time.Minute) + require.ErrorIs(t, retryErr, ErrResourceLeaseHeld) +} + +func TestResourceLeaseCanBeAcquiredFromInactiveOwnerAfterRevalidationInterval(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store1 := openTestStore(t, ctx, storePath) + store2 := openTestStore(t, ctx, storePath) + + currentProcess, currentProcessErr := process.This() + require.NoError(t, currentProcessErr) + staleOwner, staleOwnerErr := normalizeResourceLeaseOwner(process.ProcessTreeItem{ + Pid: currentProcess.Pid, + IdentityTime: currentProcess.IdentityTime.Add(-time.Hour), + }) + require.NoError(t, staleOwnerErr) + activeOwner, activeOwnerErr := normalizeResourceLeaseOwner(currentProcess) + require.NoError(t, activeOwnerErr) + + _, acquireErr := store1.AcquireResourceLease(ctx, testLeasableResource("container/test"), staleOwner, time.Minute) + require.NoError(t, acquireErr) + setResourceLeaseUpdatedAt(t, ctx, store1, "container/test", time.Now().UTC().Add(-time.Hour)) + + _, retryErr := store2.AcquireResourceLease(ctx, testLeasableResource("container/test"), activeOwner, time.Minute) + require.NoError(t, retryErr) +} + +func TestWithResourceLeaseDoesNotRetryHeldLease(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store1 := openTestStore(t, ctx, storePath) + store2 := openTestStore(t, ctx, storePath) + + owner1, owner1Err := testResourceLeaseOwner(t, 0) + require.NoError(t, owner1Err) + owner2, owner2Err := testResourceLeaseOwner(t, -time.Hour) + require.NoError(t, owner2Err) + + _, acquireErr := store1.AcquireResourceLease(ctx, testLeasableResource("container/test"), owner1, time.Minute) + require.NoError(t, acquireErr) + + callbackCalled := false + leaseErr := store2.WithResourceLease(ctx, testLeasableResource("container/test"), owner2, time.Minute, func(context.Context, *ResourceLease) error { + callbackCalled = true + return nil + }) + + require.ErrorIs(t, leaseErr, ErrResourceLeaseHeld) + require.False(t, callbackCalled) +} + +func TestDeleteInactiveResourceLeasesUsesOwnerProcessIdentity(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store1 := openTestStore(t, ctx, storePath) + store2 := openTestStore(t, ctx, storePath) + + currentProcess, currentProcessErr := process.This() + require.NoError(t, currentProcessErr) + activeOwner, activeOwnerErr := normalizeResourceLeaseOwner(currentProcess) + require.NoError(t, activeOwnerErr) + staleOwner, staleOwnerErr := normalizeResourceLeaseOwner(process.ProcessTreeItem{ + Pid: currentProcess.Pid, + IdentityTime: currentProcess.IdentityTime.Add(-time.Hour), + }) + require.NoError(t, staleOwnerErr) + + now := time.Now().UTC() + _, activeAcquireErr := store1.AcquireResourceLease(ctx, testLeasableResource("container/active"), activeOwner, time.Minute) + require.NoError(t, activeAcquireErr) + _, staleAcquireErr := store1.AcquireResourceLease(ctx, testLeasableResource("container/stale"), staleOwner, time.Minute) + require.NoError(t, staleAcquireErr) + _, invalidOwnerInsertErr := store1.db.ExecContext( + ctx, + `INSERT INTO resource_locks(resource_key, owner_pid, owner_identity_time, updated_at_unix_nano) + VALUES(?, ?, ?, ?)`, + "container/invalid-owner", + process.UnknownPID, + timeString(now), + unixNano(now), + ) + require.NoError(t, invalidOwnerInsertErr) + + require.NoError(t, store1.DeleteInactiveResourceLeases(ctx)) + + otherOwner, otherOwnerErr := testResourceLeaseOwner(t, -2*time.Hour) + require.NoError(t, otherOwnerErr) + + _, activeBlockedErr := store2.AcquireResourceLease(ctx, testLeasableResource("container/active"), otherOwner, time.Minute) + require.ErrorIs(t, activeBlockedErr, ErrResourceLeaseHeld) + + _, staleReacquireErr := store2.AcquireResourceLease(ctx, testLeasableResource("container/stale"), otherOwner, time.Minute) + require.NoError(t, staleReacquireErr) + _, invalidOwnerReacquireErr := store2.AcquireResourceLease(ctx, testLeasableResource("container/invalid-owner"), otherOwner, time.Minute) + require.NoError(t, invalidOwnerReacquireErr) +} + +func TestPersistentProcessRecordRoundTrip(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, stateStoreTestTimeout) + defer cancel() + storePath := filepath.Join(t.TempDir(), "state.sqlite3") + store := openTestStore(t, ctx, storePath) + + record := PersistentProcessRecord{ + ResourceKey: "api", + LifecycleKey: "lk1", + PID: process.Pid_t(1234), + IdentityTime: time.Unix(100, 200).UTC(), + RunID: "1234", + StdOutFile: "/tmp/stdout", + StdErrFile: "/tmp/stderr", + LifecycleMetadata: `{"args":["--port","5000"]}`, + } + + require.NoError(t, store.UpsertPersistentProcess(ctx, record)) + + actual, getErr := store.GetPersistentProcess(ctx, record.ResourceKey) + require.NoError(t, getErr) + require.Equal(t, record.ResourceKey, actual.ResourceKey) + require.Equal(t, record.LifecycleKey, actual.LifecycleKey) + require.Equal(t, record.PID, actual.PID) + require.Equal(t, record.IdentityTime, actual.IdentityTime) + require.Equal(t, record.RunID, actual.RunID) + require.Equal(t, record.StdOutFile, actual.StdOutFile) + require.Equal(t, record.StdErrFile, actual.StdErrFile) + require.Equal(t, record.LifecycleMetadata, actual.LifecycleMetadata) + require.False(t, actual.UpdatedAt.IsZero()) + + require.NoError(t, store.DeletePersistentProcess(ctx, record.ResourceKey)) + + _, getErr = store.GetPersistentProcess(ctx, record.ResourceKey) + require.True(t, errors.Is(getErr, ErrPersistentProcessNotFound), "expected ErrPersistentProcessNotFound, got %v", getErr) +} diff --git a/internal/testutil/ctrlutil/apiserver_start.go b/internal/testutil/ctrlutil/apiserver_start.go index c79afa17..46ab2b4e 100644 --- a/internal/testutil/ctrlutil/apiserver_start.go +++ b/internal/testutil/ctrlutil/apiserver_start.go @@ -39,6 +39,7 @@ import ( "github.com/microsoft/dcp/internal/containers/runtimes" "github.com/microsoft/dcp/internal/dcpclient" "github.com/microsoft/dcp/internal/dcppaths" + "github.com/microsoft/dcp/internal/statestore" "github.com/microsoft/dcp/pkg/concurrency" usvc_io "github.com/microsoft/dcp/pkg/io" "github.com/microsoft/dcp/pkg/kubeconfig" @@ -216,7 +217,10 @@ func StartApiServer( usvc_io.DCP_PRESERVE_EXECUTABLE_LOGS, kubeconfig.DCP_SECURE_TOKEN, ) - env = append(env, fmt.Sprintf("%s=%s", usvc_io.DCP_SESSION_FOLDER, sessionFolder)) + env = append(env, + fmt.Sprintf("%s=%s", usvc_io.DCP_SESSION_FOLDER, sessionFolder), + fmt.Sprintf("%s=%s", statestore.DCP_STATE_STORE_PATH, filepath.Join(sessionFolder, "state.sqlite3")), + ) cmd.Env = env var stdout, stderr bytes.Buffer diff --git a/internal/testutil/ctrlutil/test_ide_runner.go b/internal/testutil/ctrlutil/test_ide_runner.go index 11f81a4c..fb5086bb 100644 --- a/internal/testutil/ctrlutil/test_ide_runner.go +++ b/internal/testutil/ctrlutil/test_ide_runner.go @@ -134,6 +134,17 @@ func (r *TestIdeRunner) StopRun(_ context.Context, runID controllers.RunID, _ lo return r.doStopRun(runID, internal_testutil.KilledProcessExitCode) } +func (r *TestIdeRunner) ReleaseRun(_ context.Context, runID controllers.RunID, _ logr.Logger) error { + r.Runs.Range(func(name types.NamespacedName, run *TestIdeRun) bool { + if run.ID == runID { + r.Runs.Delete(name) + return false + } + return true + }) + return nil +} + func (r *TestIdeRunner) SimulateSuccessfulRunStart(runID controllers.RunID, pid process.Pid_t) error { return r.SimulateRunStart( func(_ types.NamespacedName, run *TestIdeRun) bool { return run.ID == runID }, diff --git a/internal/testutil/ctrlutil/test_process_executable_runner.go b/internal/testutil/ctrlutil/test_process_executable_runner.go index 25cd966b..3851509d 100644 --- a/internal/testutil/ctrlutil/test_process_executable_runner.go +++ b/internal/testutil/ctrlutil/test_process_executable_runner.go @@ -7,6 +7,7 @@ package ctrlutil import ( "context" + "fmt" "os/exec" "sync" "time" @@ -26,6 +27,7 @@ import ( type TestProcessExecutableRunner struct { inner controllers.ExecutableRunner autoExecutions []testutil.AutoExecution + afterStartRun func(*apiv1.Executable, *controllers.ExecutableStartResult) m *sync.RWMutex } @@ -59,6 +61,13 @@ func (e *TestProcessExecutableRunner) RemoveAutoExecution(sc testutil.ProcessSea }) } +func (r *TestProcessExecutableRunner) SetAfterStartRunHook(hook func(*apiv1.Executable, *controllers.ExecutableStartResult)) { + r.m.Lock() + defer r.m.Unlock() + + r.afterStartRun = hook +} + func (r *TestProcessExecutableRunner) StartRun( ctx context.Context, exe *apiv1.Executable, @@ -77,7 +86,9 @@ func (r *TestProcessExecutableRunner) StartRun( r.m.RUnlock() if ae == nil || ae.AsynchronousStartupDelay == 0 { - return r.inner.StartRun(ctx, exe, runChangeHandler, log) + result := r.inner.StartRun(ctx, exe, runChangeHandler, log) + r.runAfterStartRunHook(exe, result) + return result } // Start a goroutine to call the underlying runner after the delay. @@ -94,7 +105,8 @@ func (r *TestProcessExecutableRunner) StartRun( // Call the underlying runner. It will call OnStartupCompleted() on the runChangeHandler. // We don't need to do anything with the result here--it will be reported by the underlying runner // via OnStartupCompleted(). - _ = r.inner.StartRun(ctx, exe, runChangeHandler, log) + result := r.inner.StartRun(ctx, exe, runChangeHandler, log) + r.runAfterStartRunHook(exe, result) }() result := controllers.NewExecutableStartResult() @@ -102,9 +114,37 @@ func (r *TestProcessExecutableRunner) StartRun( return result } +func (r *TestProcessExecutableRunner) runAfterStartRunHook(exe *apiv1.Executable, result *controllers.ExecutableStartResult) { + r.m.RLock() + hook := r.afterStartRun + r.m.RUnlock() + if hook != nil { + hook(exe, result) + } +} + // StopRun implements ExecutableRunner by delegating to the underlying runner. func (r *TestProcessExecutableRunner) StopRun(ctx context.Context, runID controllers.RunID, log logr.Logger) error { return r.inner.StopRun(ctx, runID, log) } +func (r *TestProcessExecutableRunner) ReleaseRun(ctx context.Context, runID controllers.RunID, log logr.Logger) error { + return r.inner.ReleaseRun(ctx, runID, log) +} + +func (r *TestProcessExecutableRunner) AdoptRun( + ctx context.Context, + run controllers.ExecutableRunAdoptionInfo, + runChangeHandler controllers.RunChangeHandler, + log logr.Logger, +) error { + persistentRunner, ok := r.inner.(controllers.PersistentExecutableRunner) + if !ok { + return fmt.Errorf("inner test process runner does not support persistent run adoption") + } + + return persistentRunner.AdoptRun(ctx, run, runChangeHandler, log) +} + var _ controllers.ExecutableRunner = (*TestProcessExecutableRunner)(nil) +var _ controllers.PersistentExecutableRunner = (*TestProcessExecutableRunner)(nil) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a3d35c30..6304e172 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1,7 +1,10 @@ //go:build !ignore_autogenerated // +build !ignore_autogenerated -// Copyright (c) Microsoft Corporation. All rights reserved. +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ // Code generated by openapi-gen. DO NOT EDIT. @@ -1743,6 +1746,19 @@ func schema_microsoft_dcp_api_v1_ContainerSpec(ref common.ReferenceCallback) com Format: "", }, }, + "monitorPid": { + SchemaProps: spec.SchemaProps{ + Description: "Optional parent process PID used to scope persistent Container cleanup to a process lifecycle. When set, MonitorTimestamp must also be set and Persistent must be true.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "monitorTimestamp": { + SchemaProps: spec.SchemaProps{ + Description: "Optional parent process identity timestamp used with MonitorPID to guard against PID reuse.", + Ref: ref(metav1.MicroTime{}.OpenAPIModelName()), + }, + }, "runArgs": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -1866,7 +1882,7 @@ func schema_microsoft_dcp_api_v1_ContainerSpec(ref common.ReferenceCallback) com }, }, Dependencies: []string{ - v1.ContainerBuildContext{}.OpenAPIModelName(), v1.ContainerLabel{}.OpenAPIModelName(), v1.ContainerNetworkConnectionConfig{}.OpenAPIModelName(), v1.ContainerPemCertificates{}.OpenAPIModelName(), v1.ContainerPort{}.OpenAPIModelName(), v1.CreateFileSystem{}.OpenAPIModelName(), v1.EnvVar{}.OpenAPIModelName(), v1.HealthProbe{}.OpenAPIModelName(), v1.ImageLayer{}.OpenAPIModelName(), v1.VolumeMount{}.OpenAPIModelName()}, + v1.ContainerBuildContext{}.OpenAPIModelName(), v1.ContainerLabel{}.OpenAPIModelName(), v1.ContainerNetworkConnectionConfig{}.OpenAPIModelName(), v1.ContainerPemCertificates{}.OpenAPIModelName(), v1.ContainerPort{}.OpenAPIModelName(), v1.CreateFileSystem{}.OpenAPIModelName(), v1.EnvVar{}.OpenAPIModelName(), v1.HealthProbe{}.OpenAPIModelName(), v1.ImageLayer{}.OpenAPIModelName(), v1.VolumeMount{}.OpenAPIModelName(), metav1.MicroTime{}.OpenAPIModelName()}, } } @@ -2906,6 +2922,13 @@ func schema_microsoft_dcp_api_v1_ExecutableSpec(ref common.ReferenceCallback) co Ref: ref(v1.AmbientEnvironment{}.OpenAPIModelName()), }, }, + "start": { + SchemaProps: spec.SchemaProps{ + Description: "Should the controller attempt to start the Executable?", + Type: []string{"boolean"}, + Format: "", + }, + }, "stop": { SchemaProps: spec.SchemaProps{ Description: "Should the controller attempt to stop the Executable", @@ -2913,6 +2936,33 @@ func schema_microsoft_dcp_api_v1_ExecutableSpec(ref common.ReferenceCallback) co Format: "", }, }, + "persistent": { + SchemaProps: spec.SchemaProps{ + Description: "Should this Executable be created and persisted between DCP runs?", + Type: []string{"boolean"}, + Format: "", + }, + }, + "lifecycleKey": { + SchemaProps: spec.SchemaProps{ + Description: "Optional key used to identify if an existing persistent Executable process should be reused. If not set, the controller will calculate a key based on a hash of specific fields in the ExecutableSpec.", + Type: []string{"string"}, + Format: "", + }, + }, + "monitorPid": { + SchemaProps: spec.SchemaProps{ + Description: "Optional parent process PID used to scope persistent Executable cleanup to a process lifecycle. When set, MonitorTimestamp must also be set and Persistent must be true.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "monitorTimestamp": { + SchemaProps: spec.SchemaProps{ + Description: "Optional parent process identity timestamp used with MonitorPID to guard against PID reuse.", + Ref: ref(metav1.MicroTime{}.OpenAPIModelName()), + }, + }, "healthProbes": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -2943,7 +2993,7 @@ func schema_microsoft_dcp_api_v1_ExecutableSpec(ref common.ReferenceCallback) co }, }, Dependencies: []string{ - v1.AmbientEnvironment{}.OpenAPIModelName(), v1.EnvVar{}.OpenAPIModelName(), v1.ExecutablePemCertificates{}.OpenAPIModelName(), v1.HealthProbe{}.OpenAPIModelName()}, + v1.AmbientEnvironment{}.OpenAPIModelName(), v1.EnvVar{}.OpenAPIModelName(), v1.ExecutablePemCertificates{}.OpenAPIModelName(), v1.HealthProbe{}.OpenAPIModelName(), metav1.MicroTime{}.OpenAPIModelName()}, } } diff --git a/pkg/io/file_unix.go b/pkg/io/file_unix.go index 61dfd7e6..4518fb27 100644 --- a/pkg/io/file_unix.go +++ b/pkg/io/file_unix.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package io import "os" diff --git a/pkg/io/file_windows.go b/pkg/io/file_windows.go index 7cd5e546..ff65f4a9 100644 --- a/pkg/io/file_windows.go +++ b/pkg/io/file_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package io import ( @@ -183,7 +181,7 @@ func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { } // Write to a file on Windows. If the process is running as an administrator, we want to ensure that -// the file is only readable by other elevated processes. If not running as an administrator, we +// the file is only readable by other elevated processes. If not running as administrator, we // simply use the standard os.WriteFile function. func WriteFile(name string, data []byte, perm os.FileMode) error { file, err := OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) diff --git a/pkg/io/restricted_directory.go b/pkg/io/restricted_directory.go new file mode 100644 index 00000000..fdd84ee6 --- /dev/null +++ b/pkg/io/restricted_directory.go @@ -0,0 +1,64 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package io + +import ( + "errors" + "fmt" + "os" +) + +// EnsureRestrictedDirectory ensures the final path exists as a real directory owned by the +// current user, then applies the requested permissions. Parent directories are not created. +func EnsureRestrictedDirectory(dir string, perm os.FileMode) error { + info, statErr := os.Lstat(dir) + if errors.Is(statErr, os.ErrNotExist) { + if mkdirErr := os.Mkdir(dir, perm); mkdirErr != nil { + return fmt.Errorf("could not create directory '%s': %w", dir, mkdirErr) + } + info, statErr = os.Lstat(dir) + } + if statErr != nil { + return fmt.Errorf("could not inspect directory '%s': %w", dir, statErr) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("directory '%s' is a symlink", dir) + } + if !info.IsDir() { + return fmt.Errorf("path '%s' is not a directory", dir) + } + if ownershipErr := validateRestrictedDirectoryOwner(dir, info); ownershipErr != nil { + return fmt.Errorf("directory '%s' has invalid ownership: %w", dir, ownershipErr) + } + if chmodErr := restrictRestrictedDirectory(dir, perm); chmodErr != nil { + return fmt.Errorf("could not restrict directory '%s': %w", dir, chmodErr) + } + + return nil +} + +// ValidateRestrictedDirectory verifies that the final path exists as a real directory owned +// by the current user with the requested permissions. It does not create or chmod the directory. +func ValidateRestrictedDirectory(dir string, perm os.FileMode) error { + info, statErr := os.Lstat(dir) + if statErr != nil { + return fmt.Errorf("could not inspect directory '%s': %w", dir, statErr) + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("directory '%s' is a symlink", dir) + } + if !info.IsDir() { + return fmt.Errorf("path '%s' is not a directory", dir) + } + if ownershipErr := validateRestrictedDirectoryOwner(dir, info); ownershipErr != nil { + return fmt.Errorf("directory '%s' has invalid ownership: %w", dir, ownershipErr) + } + if modeErr := validateRestrictedDirectoryMode(dir, info, perm); modeErr != nil { + return fmt.Errorf("directory '%s' has invalid permissions: %w", dir, modeErr) + } + + return nil +} diff --git a/pkg/io/restricted_directory_test.go b/pkg/io/restricted_directory_test.go new file mode 100644 index 00000000..e2868172 --- /dev/null +++ b/pkg/io/restricted_directory_test.go @@ -0,0 +1,65 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package io_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" + + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" +) + +func TestEnsureRestrictedDirectoryRejectsSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires additional privileges on Windows") + } + + targetDir := t.TempDir() + linkPath := filepath.Join(t.TempDir(), "restricted-dir") + require.NoError(t, os.Symlink(targetDir, linkPath)) + + ensureErr := usvc_io.EnsureRestrictedDirectory(linkPath, osutil.PermissionOnlyOwnerReadWriteTraverse) + + require.Error(t, ensureErr) + require.Contains(t, ensureErr.Error(), "is a symlink") +} + +func TestEnsureRestrictedDirectoryRestrictsExistingDirectory(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "restricted-dir") + require.NoError(t, os.Mkdir(outputDir, osutil.PermissionDirectoryOthersRead)) + + require.NoError(t, usvc_io.EnsureRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) + + info, statErr := os.Lstat(outputDir) + require.NoError(t, statErr) + require.True(t, info.IsDir()) + if runtime.GOOS != "windows" { + require.Equal(t, osutil.PermissionOnlyOwnerReadWriteTraverse, info.Mode().Perm()) + } +} + +func TestValidateRestrictedDirectoryRejectsPermissiveDirectoryWithoutRestricting(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows directory permissions are not represented by Unix mode bits") + } + + outputDir := filepath.Join(t.TempDir(), "restricted-dir") + require.NoError(t, os.Mkdir(outputDir, osutil.PermissionDirectoryOthersRead)) + require.NoError(t, os.Chmod(outputDir, osutil.PermissionDirectoryOthersRead)) + + validateErr := usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse) + + require.Error(t, validateErr) + require.Contains(t, validateErr.Error(), "invalid permissions") + info, statErr := os.Lstat(outputDir) + require.NoError(t, statErr) + require.Equal(t, osutil.PermissionDirectoryOthersRead, info.Mode().Perm()) +} diff --git a/pkg/io/restricted_directory_unix.go b/pkg/io/restricted_directory_unix.go new file mode 100644 index 00000000..530f479e --- /dev/null +++ b/pkg/io/restricted_directory_unix.go @@ -0,0 +1,37 @@ +//go:build !windows + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package io + +import ( + "fmt" + "os" + "syscall" +) + +func validateRestrictedDirectoryOwner(_ string, info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("could not determine owner") + } + expectedUID := uint32(os.Geteuid()) + if stat.Uid != expectedUID { + return fmt.Errorf("owner uid is %d, expected %d", stat.Uid, expectedUID) + } + return nil +} + +func restrictRestrictedDirectory(dir string, perm os.FileMode) error { + return os.Chmod(dir, perm) +} + +func validateRestrictedDirectoryMode(_ string, info os.FileInfo, perm os.FileMode) error { + if actualPerm := info.Mode().Perm(); actualPerm != perm { + return fmt.Errorf("permissions are %s, expected %s", actualPerm, perm) + } + return nil +} diff --git a/pkg/io/restricted_directory_windows.go b/pkg/io/restricted_directory_windows.go new file mode 100644 index 00000000..58951c09 --- /dev/null +++ b/pkg/io/restricted_directory_windows.go @@ -0,0 +1,300 @@ +//go:build windows + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package io + +import ( + "fmt" + "os" + "unsafe" + + "github.com/microsoft/dcp/pkg/osutil" + "golang.org/x/sys/windows" +) + +// TOKEN_OWNER is the native TokenOwner result layout: a single PSID Owner +// field. x/sys/windows exposes the TokenOwner info class, but not the +// corresponding struct or a GetTokenOwner helper. +type tokenOwner struct { + Owner *windows.SID +} + +type restrictedDirectoryPrincipals struct { + tokenUser *windows.SID + tokenOwner *windows.SID + system *windows.SID + admins *windows.SID +} + +const ( + restrictedDirectoryAccessMask windows.ACCESS_MASK = windows.STANDARD_RIGHTS_ALL | windows.GENERIC_ALL + + // FILE_DELETE_CHILD is directory-specific and is not exposed by x/sys/windows. + restrictedDirectoryDeleteChildAccess windows.ACCESS_MASK = 0x00000040 + + restrictedDirectoryDangerousAccessMask windows.ACCESS_MASK = windows.FILE_WRITE_DATA | + windows.FILE_APPEND_DATA | + windows.FILE_WRITE_EA | + windows.FILE_WRITE_ATTRIBUTES | + restrictedDirectoryDeleteChildAccess | + windows.DELETE | + windows.WRITE_DAC | + windows.WRITE_OWNER | + windows.MAXIMUM_ALLOWED | + windows.GENERIC_WRITE | + windows.GENERIC_ALL +) + +func validateRestrictedDirectoryOwner(dir string, _ os.FileInfo) error { + var processToken windows.Token + if tokenErr := windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &processToken); tokenErr != nil { + return fmt.Errorf("could not open process token: %w", tokenErr) + } + defer processToken.Close() + + tokenUser, tokenUserErr := processToken.GetTokenUser() + if tokenUserErr != nil { + return fmt.Errorf("could not get process token user: %w", tokenUserErr) + } + + securityDescriptor, securityDescriptorErr := windows.GetNamedSecurityInfo(dir, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION) + if securityDescriptorErr != nil { + return fmt.Errorf("could not get directory security descriptor: %w", securityDescriptorErr) + } + if securityDescriptor == nil { + return fmt.Errorf("directory security descriptor is missing") + } + owner, _, ownerErr := securityDescriptor.Owner() + if ownerErr != nil { + return fmt.Errorf("could not get directory owner: %w", ownerErr) + } + // Elevated Windows tokens can create objects owned by the token owner SID + // (for example, Administrators) rather than the token user SID. Accept both + // so that a secure directory created by the current token is not rejected. + ownerMatchesTokenOwner, tokenOwnerErr := tokenOwnerMatches(processToken, owner) + if tokenOwnerErr != nil { + return fmt.Errorf("could not compare process token owner: %w", tokenOwnerErr) + } + if !windows.EqualSid(owner, tokenUser.User.Sid) && !ownerMatchesTokenOwner { + return fmt.Errorf("directory owner does not match current user or token owner") + } + + return nil +} + +func restrictRestrictedDirectory(dir string, perm os.FileMode) error { + if perm != osutil.PermissionOnlyOwnerReadWriteTraverse { + return fmt.Errorf("unsupported restricted directory permissions %s", perm) + } + + principals, principalErr := currentRestrictedDirectoryPrincipals() + if principalErr != nil { + return principalErr + } + acl, aclErr := restrictedDirectoryACL(principals) + if aclErr != nil { + return aclErr + } + + if setErr := windows.SetNamedSecurityInfo( + dir, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, + nil, + nil, + acl, + nil, + ); setErr != nil { + return fmt.Errorf("could not set restricted directory dacl: %w", setErr) + } + return nil +} + +func validateRestrictedDirectoryMode(dir string, _ os.FileInfo, perm os.FileMode) error { + if perm != osutil.PermissionOnlyOwnerReadWriteTraverse { + return fmt.Errorf("unsupported restricted directory permissions %s", perm) + } + + securityDescriptor, securityDescriptorErr := windows.GetNamedSecurityInfo( + dir, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + if securityDescriptorErr != nil { + return fmt.Errorf("could not get directory security descriptor: %w", securityDescriptorErr) + } + if securityDescriptor == nil { + return fmt.Errorf("directory security descriptor is missing") + } + + dacl, _, daclErr := securityDescriptor.DACL() + if daclErr != nil { + return fmt.Errorf("could not get directory dacl: %w", daclErr) + } + if dacl == nil { + return fmt.Errorf("directory dacl is empty") + } + + principals, principalErr := currentRestrictedDirectoryPrincipals() + if principalErr != nil { + return principalErr + } + allowedSIDs, allowedSIDErr := allowedRestrictedDirectoryValidationSIDs(principals) + if allowedSIDErr != nil { + return allowedSIDErr + } + for i := uint16(0); i < dacl.AceCount; i++ { + var ace *windows.ACCESS_ALLOWED_ACE + if aceErr := windows.GetAce(dacl, uint32(i), &ace); aceErr != nil { + return fmt.Errorf("could not inspect directory dacl entry %d: %w", i, aceErr) + } + switch ace.Header.AceType { + case windows.ACCESS_ALLOWED_ACE_TYPE: + aceSid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if !sidMatchesAny(aceSid, allowedSIDs) && aceHasDangerousAccess(ace.Mask) { + return fmt.Errorf("directory dacl grants write or control access to disallowed principal %s", aceSid.String()) + } + case windows.ACCESS_DENIED_ACE_TYPE: + continue + default: + return fmt.Errorf("directory dacl contains unsupported access entry type %d", ace.Header.AceType) + } + } + + return nil +} + +func aceHasDangerousAccess(accessMask windows.ACCESS_MASK) bool { + return accessMask&restrictedDirectoryDangerousAccessMask != 0 +} + +func restrictedDirectoryACL(principals restrictedDirectoryPrincipals) (*windows.ACL, error) { + explicitEntries := make([]windows.EXPLICIT_ACCESS, 0, 4) + for _, sid := range uniqueRestrictedDirectoryPrincipals(principals) { + explicitEntries = append(explicitEntries, windows.EXPLICIT_ACCESS{ + AccessPermissions: restrictedDirectoryAccessMask, + AccessMode: windows.GRANT_ACCESS, + Inheritance: windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT, + Trustee: windows.TRUSTEE{ + TrusteeForm: windows.TRUSTEE_IS_SID, + TrusteeType: windows.TRUSTEE_IS_UNKNOWN, + TrusteeValue: windows.TrusteeValueFromSID(sid), + }, + }) + } + + acl, aclErr := windows.ACLFromEntries(explicitEntries, nil) + if aclErr != nil { + return nil, fmt.Errorf("could not create restricted directory dacl: %w", aclErr) + } + return acl, nil +} + +func currentRestrictedDirectoryPrincipals() (restrictedDirectoryPrincipals, error) { + var processToken windows.Token + if tokenErr := windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &processToken); tokenErr != nil { + return restrictedDirectoryPrincipals{}, fmt.Errorf("could not open process token: %w", tokenErr) + } + defer processToken.Close() + + tokenUser, tokenUserErr := processToken.GetTokenUser() + if tokenUserErr != nil { + return restrictedDirectoryPrincipals{}, fmt.Errorf("could not get process token user: %w", tokenUserErr) + } + tokenUserSID, tokenUserCopyErr := tokenUser.User.Sid.Copy() + if tokenUserCopyErr != nil { + return restrictedDirectoryPrincipals{}, fmt.Errorf("could not copy process token user sid: %w", tokenUserCopyErr) + } + tokenOwnerPrincipal, tokenOwnerErr := tokenOwnerSID(processToken) + if tokenOwnerErr != nil { + return restrictedDirectoryPrincipals{}, fmt.Errorf("could not get process token owner: %w", tokenOwnerErr) + } + systemSID, systemSIDErr := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + if systemSIDErr != nil { + return restrictedDirectoryPrincipals{}, fmt.Errorf("could not get local system sid: %w", systemSIDErr) + } + adminSID, adminSIDErr := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + if adminSIDErr != nil { + return restrictedDirectoryPrincipals{}, fmt.Errorf("could not get administrators sid: %w", adminSIDErr) + } + + return restrictedDirectoryPrincipals{ + tokenUser: tokenUserSID, + tokenOwner: tokenOwnerPrincipal, + system: systemSID, + admins: adminSID, + }, nil +} + +func uniqueRestrictedDirectoryPrincipals(principals restrictedDirectoryPrincipals) []*windows.SID { + var uniqueSIDs []*windows.SID + for _, sid := range []*windows.SID{principals.tokenUser, principals.tokenOwner, principals.system, principals.admins} { + if sid != nil && !sidMatchesAny(sid, uniqueSIDs) { + uniqueSIDs = append(uniqueSIDs, sid) + } + } + return uniqueSIDs +} + +func allowedRestrictedDirectoryValidationSIDs(principals restrictedDirectoryPrincipals) ([]*windows.SID, error) { + allowedSIDs := uniqueRestrictedDirectoryPrincipals(principals) + for _, sidType := range []windows.WELL_KNOWN_SID_TYPE{ + windows.WinCreatorOwnerSid, + windows.WinCreatorOwnerRightsSid, + } { + sid, sidErr := windows.CreateWellKnownSid(sidType) + if sidErr != nil { + return nil, fmt.Errorf("could not get well-known sid %d: %w", sidType, sidErr) + } + if !sidMatchesAny(sid, allowedSIDs) { + allowedSIDs = append(allowedSIDs, sid) + } + } + return allowedSIDs, nil +} + +func sidMatchesAny(sid *windows.SID, candidates []*windows.SID) bool { + for _, candidate := range candidates { + if windows.EqualSid(sid, candidate) { + return true + } + } + return false +} + +func tokenOwnerMatches(token windows.Token, owner *windows.SID) (bool, error) { + tokenOwnerPrincipal, tokenOwnerErr := tokenOwnerSID(token) + if tokenOwnerErr != nil { + return false, tokenOwnerErr + } + + return windows.EqualSid(owner, tokenOwnerPrincipal), nil +} + +func tokenOwnerSID(token windows.Token) (*windows.SID, error) { + var requiredLength uint32 + tokenOwnerErr := windows.GetTokenInformation(token, windows.TokenOwner, nil, 0, &requiredLength) + if tokenOwnerErr != windows.ERROR_INSUFFICIENT_BUFFER { + return nil, tokenOwnerErr + } + + buffer := make([]byte, requiredLength) + tokenOwnerErr = windows.GetTokenInformation(token, windows.TokenOwner, &buffer[0], uint32(len(buffer)), &requiredLength) + if tokenOwnerErr != nil { + return nil, tokenOwnerErr + } + + // GetTokenInformation(TokenOwner) fills the buffer with a native + // TOKEN_OWNER structure. Cast the start of the buffer to that layout so we + // can read the returned Owner SID. + tokenOwnerInfo := (*tokenOwner)(unsafe.Pointer(&buffer[0])) + tokenOwnerCopy, tokenOwnerCopyErr := tokenOwnerInfo.Owner.Copy() + if tokenOwnerCopyErr != nil { + return nil, tokenOwnerCopyErr + } + return tokenOwnerCopy, nil +} diff --git a/pkg/io/restricted_directory_windows_test.go b/pkg/io/restricted_directory_windows_test.go new file mode 100644 index 00000000..67325611 --- /dev/null +++ b/pkg/io/restricted_directory_windows_test.go @@ -0,0 +1,147 @@ +//go:build windows + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package io_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" + + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" +) + +func TestEnsureRestrictedDirectoryAppliesProtectedDACL(t *testing.T) { + outputDir := t.TempDir() + + require.NoError(t, usvc_io.EnsureRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) + + require.NoError(t, usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) +} + +func TestValidateRestrictedDirectoryRejectsWorldAccess(t *testing.T) { + outputDir := t.TempDir() + worldSID, worldSIDErr := windows.CreateWellKnownSid(windows.WinWorldSid) + require.NoError(t, worldSIDErr) + setDirectoryDACL(t, outputDir, true, []windows.EXPLICIT_ACCESS{ + directoryAccessEntry(t, currentUserSID(t), windows.STANDARD_RIGHTS_ALL|windows.GENERIC_ALL, windows.GRANT_ACCESS, 0), + directoryAccessEntry(t, worldSID, windows.STANDARD_RIGHTS_ALL|windows.GENERIC_ALL, windows.GRANT_ACCESS, windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT), + }) + + validateErr := usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse) + + require.ErrorContains(t, validateErr, "write or control access to disallowed principal") +} + +func TestValidateRestrictedDirectoryAllowsInheritedReadOnlyAccess(t *testing.T) { + worldSID, worldSIDErr := windows.CreateWellKnownSid(windows.WinWorldSid) + require.NoError(t, worldSIDErr) + outputDir := directoryWithInheritedAccess(t, worldSID, windows.FILE_GENERIC_READ|windows.FILE_GENERIC_EXECUTE) + + require.NoError(t, usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) +} + +func TestValidateRestrictedDirectoryRejectsInheritedWorldWriteAccess(t *testing.T) { + worldSID, worldSIDErr := windows.CreateWellKnownSid(windows.WinWorldSid) + require.NoError(t, worldSIDErr) + outputDir := directoryWithInheritedAccess(t, worldSID, windows.FILE_GENERIC_WRITE) + + validateErr := usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse) + + require.ErrorContains(t, validateErr, "write or control access to disallowed principal") +} + +func TestValidateRestrictedDirectoryAllowsDenyAccessEntry(t *testing.T) { + outputDir := t.TempDir() + worldSID, worldSIDErr := windows.CreateWellKnownSid(windows.WinWorldSid) + require.NoError(t, worldSIDErr) + setDirectoryDACL(t, outputDir, true, []windows.EXPLICIT_ACCESS{ + directoryAccessEntry(t, currentUserSID(t), windows.STANDARD_RIGHTS_ALL|windows.GENERIC_ALL, windows.GRANT_ACCESS, 0), + directoryAccessEntry(t, worldSID, windows.FILE_WRITE_DATA, windows.DENY_ACCESS, windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT), + }) + + require.NoError(t, usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) +} + +func TestValidateRestrictedDirectoryAllowsCreatorOwnerAccessEntry(t *testing.T) { + outputDir := t.TempDir() + creatorOwnerSID, creatorOwnerSIDErr := windows.CreateWellKnownSid(windows.WinCreatorOwnerSid) + require.NoError(t, creatorOwnerSIDErr) + setDirectoryDACL(t, outputDir, true, []windows.EXPLICIT_ACCESS{ + directoryAccessEntry(t, currentUserSID(t), windows.STANDARD_RIGHTS_ALL|windows.GENERIC_ALL, windows.GRANT_ACCESS, 0), + directoryAccessEntry(t, creatorOwnerSID, windows.STANDARD_RIGHTS_ALL|windows.GENERIC_ALL, windows.GRANT_ACCESS, windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT), + }) + + require.NoError(t, usvc_io.ValidateRestrictedDirectory(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) +} + +func currentUserSID(t *testing.T) *windows.SID { + t.Helper() + var processToken windows.Token + tokenErr := windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &processToken) + require.NoError(t, tokenErr) + defer processToken.Close() + + tokenUser, tokenUserErr := processToken.GetTokenUser() + require.NoError(t, tokenUserErr) + userSID, copyErr := tokenUser.User.Sid.Copy() + require.NoError(t, copyErr) + return userSID +} + +func directoryAccessEntry(t *testing.T, sid *windows.SID, accessPermissions windows.ACCESS_MASK, accessMode windows.ACCESS_MODE, inheritance uint32) windows.EXPLICIT_ACCESS { + t.Helper() + return windows.EXPLICIT_ACCESS{ + AccessPermissions: accessPermissions, + AccessMode: accessMode, + Inheritance: inheritance, + Trustee: windows.TRUSTEE{ + TrusteeForm: windows.TRUSTEE_IS_SID, + TrusteeType: windows.TRUSTEE_IS_UNKNOWN, + TrusteeValue: windows.TrusteeValueFromSID(sid), + }, + } +} + +func setDirectoryDACL(t *testing.T, outputDir string, protected bool, entries []windows.EXPLICIT_ACCESS) { + t.Helper() + acl, aclErr := windows.ACLFromEntries(entries, nil) + require.NoError(t, aclErr) + securityInfo := windows.SECURITY_INFORMATION(windows.DACL_SECURITY_INFORMATION) + if protected { + securityInfo |= windows.PROTECTED_DACL_SECURITY_INFORMATION + } + require.NoError(t, windows.SetNamedSecurityInfo( + outputDir, + windows.SE_FILE_OBJECT, + securityInfo, + nil, + nil, + acl, + nil, + )) + t.Cleanup(func() { + _ = os.Chmod(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse) + }) +} + +func directoryWithInheritedAccess(t *testing.T, sid *windows.SID, accessPermissions windows.ACCESS_MASK) string { + t.Helper() + parentDir := t.TempDir() + setDirectoryDACL(t, parentDir, false, []windows.EXPLICIT_ACCESS{ + directoryAccessEntry(t, currentUserSID(t), windows.STANDARD_RIGHTS_ALL|windows.GENERIC_ALL, windows.GRANT_ACCESS, windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT), + directoryAccessEntry(t, sid, accessPermissions, windows.GRANT_ACCESS, windows.SUB_CONTAINERS_AND_OBJECTS_INHERIT), + }) + + outputDir := filepath.Join(parentDir, "child") + require.NoError(t, os.Mkdir(outputDir, osutil.PermissionOnlyOwnerReadWriteTraverse)) + return outputDir +} diff --git a/pkg/osutil/osutil_unix.go b/pkg/osutil/osutil_unix.go index 2a0e4c97..7e01be8a 100644 --- a/pkg/osutil/osutil_unix.go +++ b/pkg/osutil/osutil_unix.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package osutil import ( diff --git a/pkg/osutil/osutil_windows.go b/pkg/osutil/osutil_windows.go index 78ef9998..cb414ce8 100644 --- a/pkg/osutil/osutil_windows.go +++ b/pkg/osutil/osutil_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package osutil import ( diff --git a/pkg/process/os_executor_unix.go b/pkg/process/os_executor_unix.go index c4e220b4..78acfefe 100644 --- a/pkg/process/os_executor_unix.go +++ b/pkg/process/os_executor_unix.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process import ( diff --git a/pkg/process/os_executor_windows.go b/pkg/process/os_executor_windows.go index 98ba20ec..d60be570 100644 --- a/pkg/process/os_executor_windows.go +++ b/pkg/process/os_executor_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process import ( diff --git a/pkg/process/process_unix_test.go b/pkg/process/process_unix_test.go index e6b5d9b3..77826765 100644 --- a/pkg/process/process_unix_test.go +++ b/pkg/process/process_unix_test.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process_test import ( diff --git a/pkg/process/process_util.go b/pkg/process/process_util.go index f773c77e..56e1f1ff 100644 --- a/pkg/process/process_util.go +++ b/pkg/process/process_util.go @@ -29,6 +29,13 @@ type ProcessTreeItem struct { IdentityTime time.Time // Used to distinguish between different instances of processes with the same PID, may not be a valid wall-clock time. } +func FormatIdentityTime(identityTime time.Time) string { + if identityTime.IsZero() { + return "" + } + return formatIdentityTime(identityTime) +} + var ( This func() (ProcessTreeItem, error) diff --git a/pkg/process/process_util_linux.go b/pkg/process/process_util_linux.go index 329c9b8f..8017e732 100644 --- a/pkg/process/process_util_linux.go +++ b/pkg/process/process_util_linux.go @@ -1,16 +1,15 @@ +//go:build linux + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build linux - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process import ( "bytes" + "fmt" "os" "path/filepath" "strconv" @@ -61,6 +60,10 @@ func processIdentityTime(proc *ps.Process) time.Time { return time.Time{}.Add(time.Duration(startTimeMs) * time.Millisecond) } +func formatIdentityTime(identityTime time.Time) string { + return fmt.Sprintf("%dms-since-boot", identityTime.Sub(time.Time{})/time.Millisecond) +} + func init() { clkTck, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) // ignore errors diff --git a/pkg/process/process_util_linux_test.go b/pkg/process/process_util_linux_test.go new file mode 100644 index 00000000..ce58eb18 --- /dev/null +++ b/pkg/process/process_util_linux_test.go @@ -0,0 +1,23 @@ +//go:build linux + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package process + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestFormatIdentityTimeLinuxUsesDurationSinceBoot(t *testing.T) { + t.Parallel() + + identityTime := time.Time{}.Add(1234 * time.Millisecond) + + require.Equal(t, "1234ms-since-boot", FormatIdentityTime(identityTime)) +} diff --git a/pkg/process/process_util_other.go b/pkg/process/process_util_other.go index b246d806..555bfcb9 100644 --- a/pkg/process/process_util_other.go +++ b/pkg/process/process_util_other.go @@ -1,17 +1,16 @@ +//go:build !linux + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !linux - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process import ( "time" + "github.com/microsoft/dcp/pkg/osutil" ps "github.com/shirou/gopsutil/v4/process" ) @@ -23,3 +22,7 @@ func processIdentityTime(proc *ps.Process) time.Time { return time.UnixMilli(createTimestamp) } + +func formatIdentityTime(identityTime time.Time) string { + return identityTime.Format(osutil.RFC3339MiliTimestampFormat) +} diff --git a/pkg/process/process_util_unix.go b/pkg/process/process_util_unix.go index 2eeec457..7f6c524d 100644 --- a/pkg/process/process_util_unix.go +++ b/pkg/process/process_util_unix.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process import ( diff --git a/pkg/process/process_util_windows.go b/pkg/process/process_util_windows.go index 30253d44..a46d2148 100644 --- a/pkg/process/process_util_windows.go +++ b/pkg/process/process_util_windows.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process import ( diff --git a/pkg/process/process_windows_test.go b/pkg/process/process_windows_test.go index 29300ee5..01d7f6c3 100644 --- a/pkg/process/process_windows_test.go +++ b/pkg/process/process_windows_test.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package process_test import ( diff --git a/pkg/security/certstore_notwindows.go b/pkg/security/certstore_notwindows.go index 1580d6c5..fcd3f7e0 100644 --- a/pkg/security/certstore_notwindows.go +++ b/pkg/security/certstore_notwindows.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package security import "fmt" diff --git a/pkg/security/certstore_notwindows_test.go b/pkg/security/certstore_notwindows_test.go index 741d3799..c175c37c 100644 --- a/pkg/security/certstore_notwindows_test.go +++ b/pkg/security/certstore_notwindows_test.go @@ -1,12 +1,10 @@ +//go:build !windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package security import ( diff --git a/pkg/security/certstore_windows.go b/pkg/security/certstore_windows.go index 3f1eb7d7..f01eae12 100644 --- a/pkg/security/certstore_windows.go +++ b/pkg/security/certstore_windows.go @@ -5,8 +5,6 @@ * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// Copyright (c) Microsoft Corporation. All rights reserved. - package security import ( diff --git a/pkg/security/certstore_windows_test.go b/pkg/security/certstore_windows_test.go index cb0183da..f845c8ab 100644 --- a/pkg/security/certstore_windows_test.go +++ b/pkg/security/certstore_windows_test.go @@ -1,12 +1,10 @@ +//go:build windows + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build windows - -// Copyright (c) Microsoft Corporation. All rights reserved. - package security import ( diff --git a/pkg/syncmap/syncmap.go b/pkg/syncmap/syncmap.go index d2ce308e..c7212708 100644 --- a/pkg/syncmap/syncmap.go +++ b/pkg/syncmap/syncmap.go @@ -15,6 +15,10 @@ func zero[T any]() T { type Map[Key comparable, Value any] sync.Map +type ComparableValueMap[Key comparable, Value comparable] struct { + Map[Key, Value] +} + func (m *Map[Key, Value]) syncMap() *sync.Map { return (*sync.Map)(m) } @@ -99,3 +103,8 @@ func zeroIfNil[T any](v any) T { return v.(T) } } + +// Deletes the value for the passed key if the current value is equal to old. +func (m *ComparableValueMap[Key, Value]) CompareAndDelete(key Key, old Value) bool { + return m.Map.syncMap().CompareAndDelete(key, old) +} diff --git a/pkg/syncmap/syncmap_test.go b/pkg/syncmap/syncmap_test.go new file mode 100644 index 00000000..862555a9 --- /dev/null +++ b/pkg/syncmap/syncmap_test.go @@ -0,0 +1,30 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package syncmap + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestComparableValueMapCompareAndDelete(t *testing.T) { + t.Parallel() + + m := ComparableValueMap[string, *int]{} + value := 42 + other := 43 + + m.Store("key", &value) + + require.False(t, m.CompareAndDelete("key", &other)) + _, found := m.Load("key") + require.True(t, found) + + require.True(t, m.CompareAndDelete("key", &value)) + _, found = m.Load("key") + require.False(t, found) +} diff --git a/test/integration/advanced_test_env.go b/test/integration/advanced_test_env.go index d5eb9120..8f09a2d7 100644 --- a/test/integration/advanced_test_env.go +++ b/test/integration/advanced_test_env.go @@ -18,6 +18,7 @@ import ( "github.com/microsoft/dcp/controllers" dcptunproto "github.com/microsoft/dcp/internal/dcptun/proto" "github.com/microsoft/dcp/internal/health" + "github.com/microsoft/dcp/internal/statestore" ctrl_testutil "github.com/microsoft/dcp/internal/testutil/ctrlutil" "github.com/microsoft/dcp/pkg/concurrency" "github.com/microsoft/dcp/pkg/process" @@ -56,6 +57,17 @@ func StartAdvancedTestEnvironment( return nil, nil, fmt.Errorf("failed to start the API server: %w", serverErr) } + stateStore, stateStoreCleanup, stateStoreErr := createTestStateStore(ctx, testTempDir) + if stateStoreErr != nil { + serverInfo.Dispose() + return nil, nil, fmt.Errorf("failed to initialize state store: %w", stateStoreErr) + } + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + if leaseOwnerErr != nil { + serverInfo.Dispose() + stateStoreCleanup() + return nil, nil, fmt.Errorf("failed to initialize state store lease owner identity: %w", leaseOwnerErr) + } pe := process.NewOSExecutor(log) exeRunner := ctrl_testutil.NewTestProcessExecutableRunner(pe) @@ -66,6 +78,7 @@ func StartAdvancedTestEnvironment( <-managerDone.Wait() serverInfo.Dispose() + stateStoreCleanup() pe.Dispose() }) @@ -84,7 +97,7 @@ func StartAdvancedTestEnvironment( ) if inclCtrl&ExecutableController != 0 { - execR := controllers.NewExecutableReconciler( + execR := controllers.NewExecutableReconcilerWithConfig( ctx, mgr.GetClient(), mgr.GetAPIReader(), @@ -93,6 +106,10 @@ func StartAdvancedTestEnvironment( apiv1.ExecutionTypeProcess: exeRunner, }, hpSet, + controllers.ExecutableReconcilerConfig{ + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + }, ) if err = execR.SetupWithManager(mgr, instanceTag+"-ExecutableReconciler"); err != nil { return nil, nil, fmt.Errorf("failed to initialize Executable reconciler: %w", err) @@ -116,13 +133,17 @@ func StartAdvancedTestEnvironment( harvester := controllers.NewResourceHarvester() go harvester.Harvest(ctx, serverInfo.ContainerOrchestrator, log.WithName("ResourceCleanup")) - networkR := controllers.NewNetworkReconciler( + networkR := controllers.NewNetworkReconcilerWithConfig( ctx, mgr.GetClient(), mgr.GetAPIReader(), log.WithName("NetworkReconciler"), serverInfo.ContainerOrchestrator, harvester, + controllers.NetworkReconcilerConfig{ + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + }, ) if err = networkR.SetupWithManager(mgr, instanceTag+"-NetworkReconciler"); err != nil { return nil, nil, fmt.Errorf("failed to initialize Network reconciler: %w", err) @@ -139,6 +160,9 @@ func StartAdvancedTestEnvironment( hpSet, controllers.ContainerReconcilerConfig{ MaxParallelContainerStarts: math.MaxUint8, + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + ProcessExecutor: pe, }, ) if err = containerR.SetupWithManager(mgr, instanceTag+"-ContainerReconciler"); err != nil { diff --git a/test/integration/container_controller_test.go b/test/integration/container_controller_test.go index 5e2e965d..d98bfb50 100644 --- a/test/integration/container_controller_test.go +++ b/test/integration/container_controller_test.go @@ -708,6 +708,7 @@ func TestNoExistingPersistentContainerDelayStart(t *testing.T) { t.Logf("Ensure Container '%s' state is 'running'...", ctr.ObjectMeta.Name) updatedCtr, inspectedCtr := ensureContainerRunning(t, ctx, updatedCtr) require.Equal(t, inspectedCtr.Status, containers.ContainerStatusRunning, "expected the container to be in 'running' state") + waitResourceLeaseReleased(t, ctx, updatedCtr) initialLifecycleKey := updatedCtr.Status.LifecycleKey diff --git a/test/integration/controllers_common_test.go b/test/integration/controllers_common_test.go index 15668661..7d890d42 100644 --- a/test/integration/controllers_common_test.go +++ b/test/integration/controllers_common_test.go @@ -26,15 +26,19 @@ import ( clientgorest "k8s.io/client-go/rest" ctrl_client "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/stretchr/testify/require" + apiv1 "github.com/microsoft/dcp/api/v1" "github.com/microsoft/dcp/internal/dcpclient" "github.com/microsoft/dcp/internal/networking" + "github.com/microsoft/dcp/internal/statestore" internal_testutil "github.com/microsoft/dcp/internal/testutil" ctrl_testutil "github.com/microsoft/dcp/internal/testutil/ctrlutil" "github.com/microsoft/dcp/pkg/commonapi" "github.com/microsoft/dcp/pkg/concurrency" usvc_io "github.com/microsoft/dcp/pkg/io" "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/process" "github.com/microsoft/dcp/pkg/resiliency" "github.com/microsoft/dcp/pkg/slices" ) @@ -46,6 +50,8 @@ var ( client ctrl_client.Client restClient *clientgorest.RESTClient containerOrchestrator *ctrl_testutil.TestContainerOrchestrator + testStateStore *statestore.Store + testResourceLeaseOwner process.ProcessTreeItem ) const pollImmediately = true // Don't wait before polling for the first time @@ -65,6 +71,8 @@ func TestMain(m *testing.M) { testProcessExecutor = teInfo.TestProcessExecutor testProcessExecutableRunner = teInfo.TestProcessExecutableRunner ideRunner = teInfo.TestIdeRunner + testStateStore = teInfo.StateStore + testResourceLeaseOwner = teInfo.ResourceLeaseOwner networking.EnableStrictMruPortHandling(log) @@ -119,6 +127,19 @@ func waitServiceReadyEx(t *testing.T, ctx context.Context, apiClient ctrl_client return updatedSvc } +func waitResourceLeaseReleased(t *testing.T, ctx context.Context, resource statestore.LeasableResource) { + t.Helper() + + err := wait.PollUntilContextCancel(ctx, waitPollInterval, pollImmediately, func(ctx context.Context) (bool, error) { + verifyErr := testStateStore.VerifyResourceLeaseHeld(ctx, resource, testResourceLeaseOwner) + if errors.Is(verifyErr, statestore.ErrResourceLeaseNotHeld) { + return true, nil + } + return false, verifyErr + }) + require.NoError(t, err) +} + func retryOnConflict[T commonapi.ObjectStruct, PT commonapi.PObjectStruct[T]]( ctx context.Context, name types.NamespacedName, diff --git a/test/integration/executable_controller_test.go b/test/integration/executable_controller_test.go index 0c3556c2..01e31826 100644 --- a/test/integration/executable_controller_test.go +++ b/test/integration/executable_controller_test.go @@ -112,6 +112,50 @@ func TestExecutableIsStarted(t *testing.T) { } } +func TestPersistentExecutableDelayStart(t *testing.T) { + t.Parallel() + ctx, cancel := testutil.GetTestContext(t, defaultIntegrationTestTimeout) + defer cancel() + + const testName = "persistent-executable-delay-start" + shouldStart := false + exe := apiv1.Executable{ + ObjectMeta: metav1.ObjectMeta{ + Name: testName, + Namespace: metav1.NamespaceNone, + }, + Spec: apiv1.ExecutableSpec{ + ExecutablePath: "/path/to/" + testName, + Persistent: true, + Start: &shouldStart, + }, + } + + t.Logf("Creating persistent Executable '%s' with Start=false", exe.ObjectMeta.Name) + require.NoError(t, client.Create(ctx, &exe), "Could not create Executable") + + t.Logf("Waiting for Executable '%s' to be observed without starting a process", exe.ObjectMeta.Name) + updatedExe := waitObjectAssumesState(t, ctx, ctrl_client.ObjectKeyFromObject(&exe), func(currentExe *apiv1.Executable) (bool, error) { + return len(currentExe.Finalizers) > 0 && currentExe.Status.State == apiv1.ExecutableStateEmpty, nil + }) + startedProcesses := testProcessExecutor.FindAll([]string{exe.Spec.ExecutablePath}, "", nil) + require.Empty(t, startedProcesses, "persistent Executable with Start=false should not start a process") + + shouldStart = true + t.Logf("Patching persistent Executable '%s' to start", exe.ObjectMeta.Name) + require.NoError(t, retryOnConflict(ctx, updatedExe.NamespacedName(), func(ctx context.Context, currentExe *apiv1.Executable) error { + exePatch := currentExe.DeepCopy() + exePatch.Spec.Start = &shouldStart + return client.Patch(ctx, exePatch, ctrl_client.MergeFromWithOptions(currentExe, ctrl_client.MergeFromWithOptimisticLock{})) + }), "Executable object could not be patched") + + t.Logf("Waiting for persistent Executable '%s' to start", exe.ObjectMeta.Name) + updatedExe = waitObjectAssumesState(t, ctx, ctrl_client.ObjectKeyFromObject(&exe), func(currentExe *apiv1.Executable) (bool, error) { + return currentExe.Status.State == apiv1.ExecutableStateRunning || currentExe.Status.State == apiv1.ExecutableStateFinished, nil + }) + waitResourceLeaseReleased(t, ctx, updatedExe) +} + // Ensure exit code of processes/run sessions are captured correctly func TestExecutableExitCodeCaptured(t *testing.T) { type testcase struct { @@ -532,6 +576,65 @@ func TestExecutableStartupFailureProcess(t *testing.T) { }) } +func TestPersistentExecutableStopsProcessWhenProcessRecordUpdateFails(t *testing.T) { + ctx, cancel := testutil.GetTestContext(t, defaultIntegrationTestTimeout) + defer cancel() + + serverInfo, teInfo, startupErr := StartTestEnvironment(ctx, ExecutableController, t.Name(), t.TempDir()) + require.NoError(t, startupErr, "Test environment could not be started") + defer func() { + cancel() + select { + case <-serverInfo.ApiServerDisposalComplete.Wait(): + case <-time.After(5 * time.Second): + } + }() + + exe := &apiv1.Executable{ + ObjectMeta: metav1.ObjectMeta{ + Name: "persistent-executable-record-update-fails", + Namespace: metav1.NamespaceNone, + }, + Spec: apiv1.ExecutableSpec{ + ExecutablePath: "/path/to/persistent-executable-record-update-fails", + Persistent: true, + }, + } + + closeErrCh := make(chan error, 1) + var closeOnce sync.Once + teInfo.TestProcessExecutableRunner.SetAfterStartRunHook(func(startedExe *apiv1.Executable, result *controllers.ExecutableStartResult) { + if startedExe.Name != exe.Name || result == nil || !result.IsSuccessfullyCompleted() { + return + } + closeOnce.Do(func() { + closeErrCh <- teInfo.StateStore.Close() + }) + }) + defer teInfo.TestProcessExecutableRunner.SetAfterStartRunHook(nil) + + require.NoError(t, serverInfo.Client.Create(ctx, exe), "Could not create persistent Executable") + + updatedExe := waitObjectAssumesStateEx(t, ctx, serverInfo.Client, ctrl_client.ObjectKeyFromObject(exe), func(currentExe *apiv1.Executable) (bool, error) { + return currentExe.Status.State == apiv1.ExecutableStateFailedToStart && + !currentExe.Status.FinishTimestamp.IsZero(), nil + }) + + select { + case closeErr := <-closeErrCh: + require.NoError(t, closeErr, "State store could not be closed") + default: + require.Fail(t, "state store was not closed before the persistent process record update") + } + + processes := teInfo.TestProcessExecutor.FindAll([]string{exe.Spec.ExecutablePath}, "", nil) + require.Len(t, processes, 1, "Expected one process to be started") + require.True(t, processes[0].Finished(), "Process should be stopped when process record update fails") + require.Equal(t, int32(internal_testutil.KilledProcessExitCode), processes[0].ExitCode, "Process should be killed when process record update fails") + require.Empty(t, updatedExe.Status.ExecutionID, "Failed start should not report a running execution ID") + require.Equal(t, apiv1.UnknownPID, updatedExe.Status.PID, "Failed start should not report a running PID") +} + // Ensure that Executable ends up in "failed to start" state with FinishTimestamp set if the run fails // Uses IDE process runner func TestExecutableStartupFailureIDE(t *testing.T) { diff --git a/test/integration/network_controller_test.go b/test/integration/network_controller_test.go index bc0ff866..b646c8c4 100644 --- a/test/integration/network_controller_test.go +++ b/test/integration/network_controller_test.go @@ -47,7 +47,8 @@ func TestNetworkCreateInstance(t *testing.T) { err := client.Create(ctx, &net) require.NoError(t, err, "could not create a ContainerNetwork object") - _ = ensureNetworkCreated(t, ctx, &net) + updatedNet := ensureNetworkCreated(t, ctx, &net) + waitResourceLeaseReleased(t, ctx, updatedNet) } func TestNetworkRemoveInstance(t *testing.T) { diff --git a/test/integration/service_controller_test_not_darwin.go b/test/integration/service_controller_test_not_darwin.go index f94ec9bc..7210528d 100644 --- a/test/integration/service_controller_test_not_darwin.go +++ b/test/integration/service_controller_test_not_darwin.go @@ -1,12 +1,10 @@ +//go:build !darwin + /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE in the project root for license information. *--------------------------------------------------------------------------------------------*/ -//go:build !darwin - -// Copyright (c) Microsoft Corporation. All rights reserved. - package integration_test import ( diff --git a/test/integration/standard_test_env.go b/test/integration/standard_test_env.go index a0e3fdc6..e044e70c 100644 --- a/test/integration/standard_test_env.go +++ b/test/integration/standard_test_env.go @@ -21,10 +21,12 @@ import ( "github.com/microsoft/dcp/internal/dcpproc" dcptunproto "github.com/microsoft/dcp/internal/dcptun/proto" "github.com/microsoft/dcp/internal/health" + "github.com/microsoft/dcp/internal/statestore" internal_testutil "github.com/microsoft/dcp/internal/testutil" "github.com/microsoft/dcp/internal/testutil/ctrlutil" ctrl_testutil "github.com/microsoft/dcp/internal/testutil/ctrlutil" "github.com/microsoft/dcp/pkg/concurrency" + "github.com/microsoft/dcp/pkg/process" "github.com/microsoft/dcp/pkg/testutil" ) @@ -34,7 +36,9 @@ type TestEnvironmentInfo struct { *ctrl_testutil.TestProcessExecutableRunner *ctrl_testutil.TestIdeRunner *ctrl_testutil.TestTunnelControlClient - Log logr.Logger + StateStore *statestore.Store + ResourceLeaseOwner process.ProcessTreeItem + Log logr.Logger } // Starts the DCP API server (separate process) and standard controllers (in-proc). @@ -61,6 +65,17 @@ func StartTestEnvironment( return nil, nil, fmt.Errorf("failed to start the API server: %w", serverErr) } + stateStore, stateStoreCleanup, stateStoreErr := createTestStateStore(ctx, testTempDir) + if stateStoreErr != nil { + serverInfo.Dispose() + return nil, nil, fmt.Errorf("failed to initialize state store: %w", stateStoreErr) + } + leaseOwner, leaseOwnerErr := statestore.CurrentResourceLeaseOwner() + if leaseOwnerErr != nil { + serverInfo.Dispose() + stateStoreCleanup() + return nil, nil, fmt.Errorf("failed to initialize state store lease owner identity: %w", leaseOwnerErr) + } pex := internal_testutil.NewTestProcessExecutor(ctx) // On Windows the process Executable runner uses the dcp stop-process-tree subcommand, so we need to simulate that. pex.InstallAutoExecution(internal_testutil.AutoExecution{ @@ -87,6 +102,7 @@ func StartTestEnvironment( log.Error(tpeCloseErr, "Failed to close the test process executor") } + stateStoreCleanup() serverInfo.Dispose() }) @@ -105,7 +121,7 @@ func StartTestEnvironment( ) if inclCtrl&ExecutableController != 0 { - execR := controllers.NewExecutableReconciler( + execR := controllers.NewExecutableReconcilerWithConfig( ctx, mgr.GetClient(), mgr.GetAPIReader(), @@ -115,6 +131,10 @@ func StartTestEnvironment( apiv1.ExecutionTypeIDE: ir, }, hpSet, + controllers.ExecutableReconcilerConfig{ + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + }, ) if err = execR.SetupWithManager(mgr, instanceTag+"-ExecutableReconciler"); err != nil { return nil, nil, fmt.Errorf("failed to initialize Executable reconciler: %w", err) @@ -138,13 +158,17 @@ func StartTestEnvironment( harvester := controllers.NewResourceHarvester() go harvester.MockHarvest(ctx, 2*time.Second, log.WithName("ResourceCleanup")) - networkR := controllers.NewNetworkReconciler( + networkR := controllers.NewNetworkReconcilerWithConfig( ctx, mgr.GetClient(), mgr.GetAPIReader(), log.WithName("NetworkReconciler"), serverInfo.ContainerOrchestrator, harvester, + controllers.NetworkReconcilerConfig{ + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + }, ) if err = networkR.SetupWithManager(mgr, instanceTag+"-NetworkReconciler"); err != nil { return nil, nil, fmt.Errorf("failed to initialize Network reconciler: %w", err) @@ -162,6 +186,9 @@ func StartTestEnvironment( controllers.ContainerReconcilerConfig{ MaxParallelContainerStarts: math.MaxUint8, ContainerStartupTimeoutOverride: 2 * time.Second, + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, + ProcessExecutor: pex, }, ) if err = containerR.SetupWithManager(mgr, instanceTag+"-ContainerReconciler"); err != nil { @@ -256,6 +283,8 @@ func StartTestEnvironment( TestProcessExecutableRunner: exeRunner, TestIdeRunner: ir, TestTunnelControlClient: tcc, + StateStore: stateStore, + ResourceLeaseOwner: leaseOwner, Log: log, } return serverInfo, teInfo, nil diff --git a/test/integration/test_env_common.go b/test/integration/test_env_common.go index 417e594c..be06ca05 100644 --- a/test/integration/test_env_common.go +++ b/test/integration/test_env_common.go @@ -5,6 +5,14 @@ package integration_test +import ( + "context" + "os" + "path/filepath" + + "github.com/microsoft/dcp/internal/statestore" +) + type IncludedController uint32 const ( @@ -23,3 +31,35 @@ const ( const ( NoSeparateWorkingDir = "" ) + +func createTestStateStore(ctx context.Context, testTempDir string) (*statestore.Store, func(), error) { + stateStoreParentDir := testTempDir + removeStateStoreDir := false + if stateStoreParentDir == NoSeparateWorkingDir { + tempDir, tempDirErr := os.MkdirTemp("", "dcp-state-store-*") + if tempDirErr != nil { + return nil, nil, tempDirErr + } + stateStoreParentDir = tempDir + removeStateStoreDir = true + } + + stateStoreDir := filepath.Join(stateStoreParentDir, "state-store") + stateStorePath := filepath.Join(stateStoreDir, "state.sqlite3") + stateStore, stateStoreErr := statestore.Open(ctx, statestore.Options{Path: stateStorePath}) + if stateStoreErr != nil { + if removeStateStoreDir { + _ = os.RemoveAll(stateStoreParentDir) + } + return nil, nil, stateStoreErr + } + + cleanup := func() { + _ = stateStore.Close() + if removeStateStoreDir { + _ = os.RemoveAll(stateStoreParentDir) + } + } + + return stateStore, cleanup, nil +}