Skip to content

Commit

Permalink
feat(support-bundle): add runHostCollectorsInPod as host remote suppo…
Browse files Browse the repository at this point in the history
…rt bundle for kots (#1307)

* feat(support-bundle): add runHostCollectorsInPod as host remote support bundle for kots

* fix merge conflict

* fix build fail

* upgrade troubleshoot version

* add kotsadm label

* go tidy

* add e2e test

* reorder

* add host spec restore

* add createHostSupportBundle for upgrade

* add e2e test

* fix upgrade fail in test

* fix test

* fix restore test fail

* reorder check support bundle spec

* test resetore

* use label to backup

* license not licenseFile

* fix support bundle spec missing apiVersion

* test build

* fix dependency

* without some C dependencies

* without some C dependencies for all go build

* without some C dependencies for go vet

* test

* fix go vet fail

* fix go vet fail

* fix go lint fail

* fix go test fail

* fix go vet fail

* fix go lint issue

---------

Co-authored-by: Ethan Mosbaugh <ethan@replicated.com>
  • Loading branch information
DexterYan and emosbaugh authored Oct 23, 2024
1 parent 0ac1eeb commit 0ca378a
Show file tree
Hide file tree
Showing 14 changed files with 1,231 additions and 13 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ jobs:
uses: golangci/golangci-lint-action@v6
with:
version: v1.60.3
args: --build-tags exclude_graphdriver_btrfs

test:
name: Unit tests
Expand Down Expand Up @@ -519,6 +520,7 @@ jobs:
- TestMultiNodeReset
- TestCollectSupportBundle
- TestUnsupportedOverrides
- TestHostCollectSupportBundleInCluster
steps:
- name: Checkout
uses: actions/checkout@v4
Expand Down
10 changes: 5 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -234,12 +234,12 @@ embedded-cluster:
unit-tests:
mkdir -p pkg/goods/bins pkg/goods/internal/bins
touch pkg/goods/bins/BUILD pkg/goods/internal/bins/BUILD # compilation will fail if no files are present
go test -v ./pkg/... ./cmd/...
go test -tags exclude_graphdriver_btrfs -v ./pkg/... ./cmd/...
$(MAKE) -C operator test

.PHONY: vet
vet: static
go vet ./...
go vet -tags exclude_graphdriver_btrfs ./...

.PHONY: e2e-tests
e2e-tests: embedded-release
Expand All @@ -266,11 +266,11 @@ clean:

.PHONY: lint
lint:
golangci-lint run -c .golangci.yml ./...
golangci-lint run -c .golangci.yml ./... --build-tags exclude_graphdriver_btrfs

.PHONY: lint-and-fix
lint-and-fix:
golangci-lint run --fix -c .golangci.yml ./...
golangci-lint run --fix -c .golangci.yml ./... --build-tags exclude_graphdriver_btrfs

.PHONY: scan
scan:
Expand All @@ -285,7 +285,7 @@ scan:
buildtools:
mkdir -p pkg/goods/bins pkg/goods/internal/bins
touch pkg/goods/bins/BUILD pkg/goods/internal/bins/BUILD # compilation will fail if no files are present
go build -o ./output/bin/buildtools ./cmd/buildtools
go build -tags exclude_graphdriver_btrfs -o ./output/bin/buildtools ./cmd/buildtools

.PHONY: list-distros
list-distros:
Expand Down
37 changes: 37 additions & 0 deletions e2e/host-support-bundle_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package e2e

import (
"testing"
"time"

"github.com/replicatedhq/embedded-cluster/e2e/cluster/docker"
)

func TestHostCollectSupportBundleInCluster(t *testing.T) {
t.Parallel()

RequireEnvVars(t, []string{"SHORT_SHA"})

tc := docker.NewCluster(&docker.ClusterInput{
T: t,
Nodes: 1,
Distro: "debian-bookworm",
LicensePath: "license.yaml",
ECBinaryPath: "../output/bin/embedded-cluster",
})
defer tc.Cleanup()

t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
line := []string{"single-node-install.sh", "cli"}
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to install embedded-cluster: %v: %s: %s", err, stdout, stderr)
}

line = []string{"collect-support-bundle-host-in-cluster.sh"}
stdout, stderr, err := tc.RunCommandOnNode(0, line)
if err != nil {
t.Fatalf("fail to collect host support bundle: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
}
6 changes: 6 additions & 0 deletions e2e/install_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,12 @@ func TestUpgradeFromReplicatedApp(t *testing.T) {
t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr)
}

line = []string{"collect-support-bundle-host-in-cluster.sh"}
stdout, stderr, err := tc.RunCommandOnNode(0, line)
if err != nil {
t.Fatalf("fail to collect host support bundle: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
}

Expand Down
6 changes: 6 additions & 0 deletions e2e/restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ func TestSingleNodeDisasterRecovery(t *testing.T) {
t.Fatalf("fail to restore the installation: %v: %s: %s", err, stdout, stderr)
}

line = []string{"collect-support-bundle-host-in-cluster.sh"}
stdout, stderr, err := tc.RunCommandOnNode(0, line)
if err != nil {
t.Fatalf("fail to collect host support bundle: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()}
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
Expand Down
14 changes: 14 additions & 0 deletions e2e/scripts/collect-support-bundle-host-in-cluster.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euox pipefail

DIR=/usr/local/bin
. $DIR/common.sh

main() {
if ! kubectl get cm -n kotsadm embedded-cluster-host-support-bundle -o yaml ; then
echo "Failed to get configmap of remote host support bundle spec for kotsadm"
return 1
fi
}

main "$@"
76 changes: 74 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ require (
github.com/replicatedhq/embedded-cluster/kinds v0.0.0
github.com/replicatedhq/embedded-cluster/utils v0.0.0
github.com/replicatedhq/kotskinds v0.0.0-20240814191029-3f677ee409a0
github.com/replicatedhq/troubleshoot v0.107.1
github.com/replicatedhq/troubleshoot v0.107.2
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.1
github.com/spf13/viper v1.19.0
Expand All @@ -47,6 +47,7 @@ require (
k8s.io/api v0.31.1
k8s.io/apimachinery v0.31.1
k8s.io/client-go v0.31.1
k8s.io/kubectl v0.31.1
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3
oras.land/oras-go/v2 v2.5.0
sigs.k8s.io/controller-runtime v0.19.0
Expand All @@ -59,14 +60,23 @@ replace (
)

require (
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/compute/metadata v0.3.0 // indirect
cloud.google.com/go/iam v1.1.7 // indirect
cloud.google.com/go/storage v1.40.0 // indirect
dario.cat/mergo v1.0.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/sprig/v3 v3.3.0 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Microsoft/hcsshim v0.12.5 // indirect
github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412 // indirect
github.com/andybalholm/brotli v1.0.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
Expand All @@ -82,25 +92,33 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
github.com/aws/smithy-go v1.22.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/chzyer/readline v1.5.1 // indirect
github.com/cilium/ebpf v0.16.0 // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/containerd/containerd v1.7.22 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
github.com/containers/ocicrypt v1.2.0 // indirect
github.com/containers/storage v1.55.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/cyphar/filepath-securejoin v0.3.1 // indirect
github.com/distribution/distribution/v3 v3.0.0-beta.1 // indirect
github.com/docker/cli v27.1.1+incompatible // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v27.1.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.8.2 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
Expand All @@ -109,46 +127,80 @@ require (
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-redis/redis/v7 v7.4.1 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-containerregistry v0.20.0 // indirect
github.com/google/go-intervals v0.0.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.3 // indirect
github.com/gorilla/handlers v1.5.2 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-getter v1.7.6 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.5.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.7.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jmoiron/sqlx v1.4.0 // indirect
github.com/k0sproject/version v0.6.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/manifoldco/promptui v0.9.0 // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/microsoft/go-mssqldb v1.7.2 // indirect
github.com/miekg/dns v1.1.62 // indirect
github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.4.0 // indirect
github.com/moby/sys/mountinfo v0.7.2 // indirect
github.com/moby/sys/user v0.3.0 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/nwaples/rardecode v1.1.2 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.20.3 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.59.1 // indirect
Expand All @@ -157,37 +209,55 @@ require (
github.com/rubenv/sql-migrate v1.7.0 // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/segmentio/ksuid v1.0.4 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/sylabs/sif/v2 v2.18.0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/tchap/go-patricia/v2 v2.3.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/vbatts/tar-split v0.11.5 // indirect
github.com/vishvananda/netlink v1.3.0 // indirect
github.com/vishvananda/netns v0.0.4 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zitadel/logging v0.6.0 // indirect
github.com/zitadel/oidc/v3 v3.30.0 // indirect
github.com/zitadel/schema v1.3.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.16 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect
go.etcd.io/etcd/client/v3 v3.5.16 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/exporters/autoexport v0.52.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
go.opentelemetry.io/otel v1.31.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.31.0 // indirect
go.opentelemetry.io/otel/sdk v1.31.0 // indirect
go.opentelemetry.io/otel/trace v1.31.0 // indirect
go.starlark.net v0.0.0-20240725214946-42030a7cedce // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/tools v0.25.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/api v0.172.0 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/grpc v1.66.2 // indirect
Expand All @@ -196,8 +266,10 @@ require (
k8s.io/apiserver v0.31.1 // indirect
k8s.io/cli-runtime v0.31.1 // indirect
k8s.io/component-base v0.31.1 // indirect
k8s.io/kubectl v0.31.1 // indirect
k8s.io/kubelet v0.31.1 // indirect
k8s.io/metrics v0.31.1 // indirect
oras.land/oras-go v1.2.6 // indirect
periph.io/x/host/v3 v3.8.2 // indirect
sigs.k8s.io/kustomize/api v0.17.2 // indirect
sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
)
Expand Down
Loading

0 comments on commit 0ca378a

Please sign in to comment.