Revert "Remove certmgr from vizier"

Summary: This reverts commit 29b5d31.

Test Plan: Clean revert, vizier should still work

Reviewers: michelle

Reviewed By: michelle

Signed-off-by: Vihang Mehta <vihang@pixielabs.ai>

Differential Revision: https://phab.corp.pixielabs.ai/D11150

GitOrigin-RevId: 044086c

.fossa.yml

@@ -812,6 +812,26 @@ analyze:
     type: go
     target: px.dev/pixie/src/vizier/messages/messagespb
     path: src/vizier/messages/messagespb
+  - name: px.dev/pixie/src/vizier/services/certmgr
+    type: go
+    target: px.dev/pixie/src/vizier/services/certmgr
+    path: src/vizier/services/certmgr
+  - name: px.dev/pixie/src/vizier/services/certmgr/certmgrenv
+    type: go
+    target: px.dev/pixie/src/vizier/services/certmgr/certmgrenv
+    path: src/vizier/services/certmgr/certmgrenv
+  - name: px.dev/pixie/src/vizier/services/certmgr/certmgrpb
+    type: go
+    target: px.dev/pixie/src/vizier/services/certmgr/certmgrpb
+    path: src/vizier/services/certmgr/certmgrpb
+  - name: px.dev/pixie/src/vizier/services/certmgr/controller
+    type: go
+    target: px.dev/pixie/src/vizier/services/certmgr/controller
+    path: src/vizier/services/certmgr/controller
+  - name: px.dev/pixie/src/vizier/services/certmgr/controllers/mock
+    type: go
+    target: px.dev/pixie/src/vizier/services/certmgr/controllers/mock
+    path: src/vizier/services/certmgr/controllers/mock
   - name: px.dev/pixie/src/vizier/services/cloud_connector
     type: go
     target: px.dev/pixie/src/vizier/services/cloud_connector

BUILD.bazel

@@ -88,6 +88,7 @@ gazelle(
 # gazelle:resolve go px.dev/pixie/src/stirling/testing/demo_apps/hipster_shop/proto //src/stirling/testing/demo_apps/hipster_shop/proto:demo_pl_go_proto
 # gazelle:resolve go px.dev/pixie/src/table_store/schemapb //src/table_store/schemapb:schema_pl_go_proto
 # gazelle:resolve go px.dev/pixie/src/vizier/messages/messagespb //src/vizier/messages/messagespb:messages_pl_go_proto
+# gazelle:resolve go px.dev/pixie/src/vizier/services/certmgr/certmgrpb //src/vizier/services/certmgr/certmgrpb:service_pl_go_proto
 # gazelle:resolve go px.dev/pixie/src/vizier/services/metadata/metadatapb //src/vizier/services/metadata/metadatapb:service_pl_go_proto
 # gazelle:resolve go px.dev/pixie/src/vizier/services/metadata/storepb //src/vizier/services/metadata/storepb:store_pl_go_proto
 # gazelle:resolve go px.dev/pixie/src/vizier/services/shared/agentpb //src/vizier/services/shared/agentpb:agent_pl_go_proto

k8s/vizier/BUILD.bazel

@@ -23,6 +23,7 @@ package(default_visibility = ["//visibility:public"])
 
 VIZIER_IMAGE_MAP = {
     "gcr.io/pixie-oss/pixie-dev/vizier/cert_provisioner_image": "//src/utils/cert_provisioner:cert_provisioner_image",
+    "gcr.io/pixie-oss/pixie-dev/vizier/certmgr_server_image": "//src/vizier/services/certmgr:certmgr_server_image",
     "gcr.io/pixie-oss/pixie-dev/vizier/cloud_connector_server_image": "//src/vizier/services/cloud_connector:cloud_connector_server_image",
     "gcr.io/pixie-oss/pixie-dev/vizier/kelvin_image": "//src/vizier/services/agent/kelvin:kelvin_image",
     "gcr.io/pixie-oss/pixie-dev/vizier/metadata_server_image": "//src/vizier/services/metadata:metadata_server_image",

k8s/vizier/base/certmgr_deployment.yaml

@@ -0,0 +1,84 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vizier-certmgr
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: vizier-certmgr
+  template:
+    metadata:
+      labels:
+        name: vizier-certmgr
+        plane: control
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: Exists
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+            - matchExpressions:
+              - key: beta.kubernetes.io/os
+                operator: Exists
+              - key: beta.kubernetes.io/os
+                operator: In
+                values:
+                - linux
+      serviceAccountName: certmgr-service-account
+      initContainers:
+      - name: nats-wait
+        image: gcr.io/pixie-oss/pixie-dev-public/curl:1.0
+        # yamllint disable
+        command: ['sh', '-c', 'set -xe;
+          URL="${PROTOCOL}://${SERVICE_NAME}:${SERVICE_PORT}${HEALTH_PATH}";
+          until [ $(curl -m 0.5 -s -o /dev/null -w "%{http_code}" -k ${URL}) -eq 200 ]; do
+            echo "waiting for ${URL}";
+            sleep 2;
+          done;']
+        env:
+        - name: SERVICE_NAME
+          value: "pl-nats-mgmt"
+        - name: SERVICE_PORT
+          value: "8222"
+        - name: HEALTH_PATH
+          value: ""
+        - name: PROTOCOL
+          value: "http"
+      containers:
+      - name: app
+        image: gcr.io/pixie-oss/pixie-dev/vizier/certmgr_server_image:latest
+        env:
+        - name: PL_JWT_SIGNING_KEY
+          valueFrom:
+            secretKeyRef:
+              key: jwt-signing-key
+              name: pl-cluster-secrets
+        - name: PL_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        envFrom:
+        - configMapRef:
+            name: pl-tls-config
+        ports:
+        - containerPort: 50900
+        volumeMounts:
+        - mountPath: /certs
+          name: certs
+        livenessProbe:
+          httpGet:
+            scheme: HTTPS
+            path: /healthz
+            port: 50900
+      volumes:
+      - name: certs
+        secret:
+          secretName: service-tls-certs

k8s/vizier/base/certmgr_role.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: certmgr-service-account
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pl-vizier-certmgr
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - secrets
+  - services
+  verbs:
+  - "*"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pl-vizier-certmgr-cluster-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pl-vizier-certmgr
+subjects:
+- kind: ServiceAccount
+  name: certmgr-service-account
+  namespace: pl
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pl-vizier-crd-certmgr-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pl-vizier-crd-role
+subjects:
+- kind: ServiceAccount
+  name: certmgr-service-account
+  namespace: pl

k8s/vizier/base/certmgr_service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: vizier-certmgr-svc
+spec:
+  type: ClusterIP
+  ports:
+  - name: tcp-http2
+    port: 50900
+    protocol: TCP
+    targetPort: 50900
+  selector:
+    name: vizier-certmgr

k8s/vizier/base/kustomization.yaml

@@ -13,6 +13,9 @@ patches:
 resources:
 - ../bootstrap
 - proxy_envoy.yaml
+- certmgr_role.yaml
+- certmgr_deployment.yaml
+- certmgr_service.yaml
 - kelvin_deployment.yaml
 - kelvin_service.yaml
 - metadata_role.yaml

skaffold/skaffold_vizier.yaml

@@ -36,6 +36,12 @@ build:
       target: //src/vizier/services/cloud_connector:cloud_connector_server_image.tar
       args:
       - --compilation_mode=dbg
+  - image: gcr.io/pixie-oss/pixie-dev/vizier/certmgr_server_image
+    context: .
+    bazel:
+      target: //src/vizier/services/certmgr:certmgr_server_image.tar
+      args:
+      - --compilation_mode=dbg
   - image: gcr.io/pixie-oss/pixie-dev/vizier/cert_provisioner_image
     context: .
     bazel:

src/vizier/services/certmgr/BUILD.bazel

@@ -0,0 +1,63 @@
+# Copyright 2018- The Pixie Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load("@io_bazel_rules_docker//container:container.bzl", "container_push")
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "certmgr_lib",
+    srcs = ["certmgr_server.go"],
+    importpath = "px.dev/pixie/src/vizier/services/certmgr",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//src/shared/services",
+        "//src/shared/services/healthz",
+        "//src/shared/services/server",
+        "//src/vizier/services/certmgr/certmgrenv",
+        "//src/vizier/services/certmgr/certmgrpb:service_pl_go_proto",
+        "//src/vizier/services/certmgr/controllers",
+        "@com_github_gofrs_uuid//:uuid",
+        "@com_github_nats_io_nats_go//:nats_go",
+        "@com_github_sirupsen_logrus//:logrus",
+        "@com_github_spf13_pflag//:pflag",
+        "@com_github_spf13_viper//:viper",
+    ],
+)
+
+go_binary(
+    name = "certmgr",
+    embed = [":certmgr_lib"],
+    visibility = ["//visibility:public"],
+)
+
+go_image(
+    name = "certmgr_server_image",
+    binary = ":certmgr",
+    visibility = [
+        "//k8s:__subpackages__",
+        "//src/vizier:__subpackages__",
+    ],
+)
+
+container_push(
+    name = "push_certmgr_server_image",
+    format = "Docker",
+    image = ":certmgr_server_image",
+    registry = "gcr.io",
+    repository = "pixie-oss/pixie-dev/vizier/certmgr_server_image",
+    tag = "{STABLE_BUILD_TAG}",
+)

src/vizier/services/certmgr/certmgr_server.go

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2018- The Pixie Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package main
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gofrs/uuid"
+	"github.com/nats-io/nats.go"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
+	"px.dev/pixie/src/shared/services"
+	"px.dev/pixie/src/shared/services/healthz"
+	"px.dev/pixie/src/shared/services/server"
+	"px.dev/pixie/src/vizier/services/certmgr/certmgrenv"
+	"px.dev/pixie/src/vizier/services/certmgr/certmgrpb"
+	"px.dev/pixie/src/vizier/services/certmgr/controllers"
+)
+
+func init() {
+	pflag.String("namespace", "pl", "The namespace of Vizier")
+	pflag.String("cluster_id", "", "The Cluster ID to use for Pixie Cloud")
+	pflag.String("nats_url", "pl-nats", "The URL of NATS")
+}
+
+func main() {
+	services.SetupService("certmgr-service", 50900)
+	services.SetupSSLClientFlags()
+	services.PostFlagSetupAndParse()
+	services.CheckServiceFlags()
+	services.CheckSSLClientFlags()
+	services.SetupServiceLogging()
+
+	flush := services.InitSentryFromCRD(viper.GetString("cluster_id"),
+		viper.GetString("namespace"))
+	defer flush()
+
+	natsWait := make(chan struct{})
+	var nc *nats.Conn
+	var err error
+
+	go func() {
+		nc, err = nats.Connect(viper.GetString("nats_url"),
+			nats.ClientCert(viper.GetString("client_tls_cert"), viper.GetString("client_tls_key")),
+			nats.RootCAs(viper.GetString("tls_ca_cert")))
+		if err != nil {
+			log.WithError(err).Fatal("Failed to connect to NATS.")
+		}
+		close(natsWait)
+	}()
+
+	select {
+	case <-natsWait:
+		log.Info("Connected to NATS")
+	case <-time.After(1 * time.Minute):
+		log.WithError(err).Fatal("Timed out: failed to connect to NATS.")
+	}
+
+	clusterID, err := uuid.FromString(viper.GetString("cluster_id"))
+	if err != nil {
+		log.WithError(err).Fatal("Failed to parse passed in cluster ID")
+	}
+
+	mux := http.NewServeMux()
+	healthz.RegisterDefaultChecks(mux)
+
+	k8sWait := make(chan struct{})
+	var k8sAPI *controllers.K8sAPIImpl
+
+	go func() {
+		k8sAPI, err = controllers.NewK8sAPI(viper.GetString("namespace"))
+		if err != nil {
+			log.WithError(err).Fatal("Failed to connect to K8S API")
+		}
+		close(k8sWait)
+	}()
+
+	select {
+	case <-k8sWait:
+		log.Info("Connected to K8s API")
+	case <-time.After(1 * time.Minute):
+		log.WithError(err).Fatal("Timed out: failed to connect to K8s API.")
+	}
+
+	env := certmgrenv.New("vizier")
+	svr := controllers.NewServer(env, clusterID, nc, k8sAPI)
+	go svr.CertRequester()
+	defer svr.StopCertRequester()
+
+	s := server.NewPLServer(env, mux)
+	certmgrpb.RegisterCertMgrServiceServer(s.GRPCServer(), svr)
+	s.Start()
+	s.StopOnInterrupt()
+}