WIP - chore(backend): remove bazel from backend code generation

backend/Dockerfile.buildapi

@@ -0,0 +1,37 @@
+# 0. Generate client code (go & json) from API protocol buffers
+FROM golang:1.13.1-stretch as generator
+ENV KFP_VERSION 1.0.0
+ENV GRPC_GATEWAY_VERSION v1.9.0
+ENV GO_SWAGGER_VERSION v0.18.0
+ENV GOLANG_PROTOBUF_VERSION v1.3.1
+ENV GRPC_VERSION v1.23.0
+ENV PROTOC_VERSION 3.14.0
+
+# Install protoc.
+RUN apt-get update -y && apt-get install -y jq sed unzip
+RUN curl -L -o protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
+RUN unzip -o protoc.zip -d /usr/ bin/protoc
+RUN unzip -o protoc.zip -d /usr/ 'include/*'
+RUN rm -f protoc.zip
+ENV PROTOCCOMPILER /usr/bin/protoc
+ENV PROTOCINCLUDE /usr/include/google/protobuf
+
+# Need grpc-gateway source code for -I in protoc command.
+WORKDIR /go/src/github.com
+RUN mkdir grpc-ecosystem && cd grpc-ecosystem && git clone --depth 1 --branch $GRPC_GATEWAY_VERSION https://github.com/grpc-ecosystem/grpc-gateway.git
+RUN mkdir grpc && git clone --depth 1 --branch $GRPC_VERSION https://github.com/grpc/grpc-go
+
+# Install protoc-gen-rpc-gateway && protoc-gen-swagger.
+RUN cd grpc-ecosystem/grpc-gateway && go mod vendor
+RUN GOBIN=/go/bin go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
+RUN GOBIN=/go/bin go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+
+# Download go-swagger binary.
+RUN curl -LO "https://github.com/go-swagger/go-swagger/releases/download/${GO_SWAGGER_VERSION}/swagger_linux_amd64"
+RUN chmod +x swagger_linux_amd64 && mv swagger_linux_amd64 /usr/bin/swagger
+
+# Need protobuf source code for -I in protoc command.
+RUN mkdir golang && cd golang && git clone --depth 1 --branch $GOLANG_PROTOBUF_VERSION https://github.com/golang/protobuf.git
+# Install protoc-gen-go.
+RUN cd golang/protobuf && go mod vendor
+RUN GOBIN=/go/bin go install github.com/golang/protobuf/protoc-gen-go

backend/README.md

@@ -35,7 +35,11 @@ Change it to point to your own build, after saving and closing the file, apiserv
 
 After making changes to proto files, the Go client libraries and swagger files
 need to be regenerated and checked-in. The `backend/api/generate_api.sh` script
-takes care of this. It should be noted that this requires [Bazel](https://bazel.build/), version 0.24.0` 
+takes care of this. In order for the generated code to be owned by root, root credentials has to be provided. 
+
+```bash
+sh generate_api.sh
+```
 
 # Visualization Server Instructions
 

backend/api/auth.proto

@@ -19,7 +19,6 @@ package api;
 
 import "google/api/annotations.proto";
 import "google/protobuf/empty.proto";
-import "backend/api/error.proto";
 import "protoc-gen-swagger/options/annotations.proto";
 
 option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = {

backend/api/build_kfp_server_api_python_package.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 #
-# Copyright 2018-2020 Google LLC
+# Copyright 2018-2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -66,10 +66,6 @@ pushd "$DIR"
 python3 setup.py --quiet sdist
 popd
 
-echo "Adding license header for generated python files in $DIR."
-go install github.com/google/addlicense
-addlicense "$DIR"
-
 echo "Run the following commands to update the package on PyPI"
 echo "python3 -m pip install twine"
 echo "python3 -m twine upload --username kubeflow-pipelines $DIR/dist/*"

backend/api/generate_api.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2018-2020 Google LLC
+# Copyright 2018-2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,124 +21,15 @@
 
 set -ex
 
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
-REPO_ROOT="$DIR/../.."
-VERSION="$(cat $REPO_ROOT/VERSION)"
-if [ -z "$VERSION" ]; then
-    echo "ERROR: $REPO_ROOT/VERSION is empty"
-    exit 1
-fi
+cd ../..
+# docker run  -it --rm \
+#     --mount type=bind,source="$(pwd)",target=/app/pipelines \
+#     builder /bin/bash
 
-BAZEL_BINDIR=$(bazel info bazel-bin)
-SWAGGER_CMD=${DIR}/../../bazel-bin/external/com_github_go_swagger/cmd/swagger/*stripped/swagger
-AUTOGEN_CMD="${DIR}/../../bazel-bin/external/com_github_mbrukman_autogen/autogen_tool"
-GENERATED_GO_PROTO_FILES="${BAZEL_BINDIR}/backend/api/api_generated_go_sources/src/github.com/kubeflow/pipelines/backend/api/go_client/*.go"
+# Generate API
+docker run  --interactive --rm \
+    --mount type=bind,source="$(pwd)",target=/go/src/github.com/kubeflow/pipelines \
+    builder /go/src/github.com/kubeflow/pipelines/backend/api/generator.sh
 
-# TODO this script should be able to be run from anywhere, not just within .../backend/api/
-
-# Delete currently generated code.
-rm -r -f ${DIR}/go_http_client/*
-rm -r -f ${DIR}/go_client/*
-
-# Build required tools.
-bazel build @com_github_mbrukman_autogen//:autogen_tool
-bazel build @com_github_go_swagger//cmd/swagger
-
-# Build .pb.go and .gw.pb.go files from the proto sources.
-bazel build //backend/api:api_generated_go_sources
-
-# Copy the generated files into the source tree and add license.
-for f in $GENERATED_GO_PROTO_FILES; do
-  target=${DIR}/go_client/$(basename ${f})
-  cp $f $target
-  chmod 766 $target
-  ${AUTOGEN_CMD} -i --no-tlc -c "Google LLC" -l apache $target
-done
-
-# Generate and copy back into source tree .swagger.json files.
-bazel build //backend/api:api_swagger
-cp ${BAZEL_BINDIR}/backend/api/*.swagger.json ${DIR}/swagger
-
-jq -s '
-    reduce .[] as $item ({}; . * $item) |
-    .info.title = "Kubeflow Pipelines API" |
-    .info.description = "This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition." |
-    .info.version = "'$VERSION'" |
-    .info.contact = { "name": "google", "email": "kubeflow-pipelines@google.com", "url": "https://www.google.com" } |
-    .info.license = { "name": "Apache 2.0", "url": "https://raw.githubusercontent.com/kubeflow/pipelines/master/LICENSE" }
-' ${DIR}/swagger/{run,job,pipeline,experiment,pipeline.upload,healthz}.swagger.json > "${DIR}/swagger/kfp_api_single_file.swagger.json"
-
-# Generate Go HTTP client from the swagger files.
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/job.swagger.json \
-  -A job \
-  --principal models.Principal \
-  -c job_client \
-  -m job_model \
-  -t ${DIR}/go_http_client
-
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/run.swagger.json \
-  -A run \
-  --principal models.Principal \
-  -c run_client \
-  -m run_model \
-  -t ${DIR}/go_http_client
-
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/experiment.swagger.json \
-  -A experiment \
-  --principal models.Principal \
-  -c experiment_client \
-  -m experiment_model \
-  -t ${DIR}/go_http_client
-
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/pipeline.upload.swagger.json \
-  -A pipeline_upload \
-  --principal models.Principal \
-  -c pipeline_upload_client \
-  -m pipeline_upload_model \
-  -t ${DIR}/go_http_client
-
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/pipeline.swagger.json \
-  -A pipeline \
-  --principal models.Principal \
-  -c pipeline_client \
-  -m pipeline_model \
-  -t ${DIR}/go_http_client
-
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/visualization.swagger.json \
-  -A visualization \
-  --principal models.Principal \
-  -c visualization_client \
-  -m visualization_model \
-  -t ${DIR}/go_http_client
-
-${SWAGGER_CMD} generate client \
-  -f ${DIR}/swagger/healthz.swagger.json \
-  -A healthz \
-  --principal models.Principal \
-  -c healthz_client \
-  -m healthz_model \
-  -t ${DIR}/go_http_client
-
-# Hack to fix an issue with go-swagger
-# See https://github.com/go-swagger/go-swagger/issues/1381 for details.
-sed -i -- 's/MaxConcurrency int64 `json:"max_concurrency,omitempty"`/MaxConcurrency int64 `json:"max_concurrency,omitempty,string"`/g' ${DIR}/go_http_client/job_model/api_job.go
-sed -i -- 's/IntervalSecond int64 `json:"interval_second,omitempty"`/IntervalSecond int64 `json:"interval_second,omitempty,string"`/g' ${DIR}/go_http_client/job_model/api_periodic_schedule.go
-sed -i -- 's/MaxConcurrency string `json:"max_concurrency,omitempty"`/MaxConcurrency int64 `json:"max_concurrency,omitempty,string"`/g' ${DIR}/go_http_client/job_model/api_job.go
-sed -i -- 's/IntervalSecond string `json:"interval_second,omitempty"`/IntervalSecond int64 `json:"interval_second,omitempty,string"`/g' ${DIR}/go_http_client/job_model/api_periodic_schedule.go
-
-# Executes the //go:generate directives in the generated code.
-go generate ./...
-
-# Add license to files in go_http_client.
-find ${DIR}/go_http_client/ -name "*.go" -exec ${AUTOGEN_CMD} -i --no-tlc -c "Google LLC" -l apache {} \;
-
-# Finally, run gazelle to add BUILD files for the generated code.
-bazel run //:gazelle
-# HACK: remove unnecessary BUILD.bazels
-rm -f "$REPO_ROOT/sdk/python/kfp/components/structures/BUILD.bazel" "$REPO_ROOT/tools/metadatastore-upgrade/BUILD.bazel"
+# Change owner to user for generate files, explanation of command: https://askubuntu.com/questions/829537/how-do-i-change-owner-to-current-user-on-folder-and-containing-folders-inside-my
+sudo find backend/api -user root -exec sudo chown $USER: {} +

backend/api/generator.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Copyright 2018-2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+export TMP_OUTPUT=/tmp
+
+# Change directory\
+cd /go/src/github.com/kubeflow/pipelines
+# Delete currently generated code.
+rm -r -f backend/api/go_http_client/*
+rm -f -f backend/api/go_client/*
+
+mkdir -p backend/api/go_client
+# Generate *.pb.go (grpc api client) from *.proto .
+${PROTOCCOMPILER} -I. -Ibackend/api \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+    -I/usr/include/ \
+    --plugin=protoc-gen-go=/go/bin/protoc-gen-go  \
+    --go_out=plugins=grpc:${TMP_OUTPUT} \
+    backend/api/*.proto
+# Generate *.pb.gw.go (grpc api rest client) from *.proto.
+${PROTOCCOMPILER} -I. -Ibackend/api \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+    -I/usr/include/ \
+    --plugin=protoc-gen-grpc-gateway=/go/bin/protoc-gen-grpc-gateway \
+    --grpc-gateway_out=logtostderr=true:${TMP_OUTPUT} \
+    backend/api/*.proto
+# Move *.pb.go and *.gw.go to go_client folder.
+cp ${TMP_OUTPUT}/github.com/kubeflow/pipelines/backend/api/go_client/* ./backend/api/go_client
+# Generate *.swagger.json from *.proto into swagger folder.
+${PROTOCCOMPILER} -I. -Ibackend/api \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+    -I/usr/include/ \
+    --plugin=protoc-gen-swagger=/go/bin/protoc-gen-swagger \
+    --swagger_out=logtostderr=true:${TMP_OUTPUT} \
+    backend/api/*.proto
+cp ${TMP_OUTPUT}/backend/api/*.swagger.json ./backend/api/swagger
+# Generate a single swagger json file from the swagger json files of all models.
+# Note: use backend/backend/api/swagger/{run,job,experiment,pipeline,pipeline.upload}.swagger.json when apt-get can install jq-1.6
+jq -s 'reduce .[] as $item ({}; . * $item) | .info.title = "Kubeflow Pipelines API" | .info.description = "This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition." | .info.version = "'$KFP_VERSION'" | .info.contact = { "name": "google", "email": "kubeflow-pipelines@google.com", "url": "https://www.google.com" } | .info.license = { "name": "Apache 2.0", "url": "https://raw.githubusercontent.com/kubeflow/pipelines/master/LICENSE" }' backend/api/swagger/run.swagger.json backend/api/swagger/job.swagger.json backend/api/swagger/experiment.swagger.json backend/api/swagger/pipeline.swagger.json backend/api/swagger/pipeline.upload.swagger.json > "backend/api/swagger/kfp_api_single_file.swagger.json"
+# Generate go_http_client from swagger json.
+mkdir -p backend/api/go_http_client
+swagger generate client \
+    -f backend/api/swagger/job.swagger.json \
+    -A job \
+    --principal models.Principal \
+    -c job_client \
+    -m job_model \
+    -t backend/api/go_http_client
+swagger generate client \
+    -f backend/api/swagger/run.swagger.json \
+    -A run \
+    --principal models.Principal \
+    -c run_client \
+    -m run_model \
+    -t backend/api/go_http_client
+swagger generate client \
+    -f backend/api/swagger/experiment.swagger.json \
+    -A experiment \
+    --principal models.Principal \
+    -c experiment_client \
+    -m experiment_model \
+    -t backend/api/go_http_client
+swagger generate client \
+    -f backend/api/swagger/pipeline.swagger.json \
+    -A pipeline \
+    --principal models.Principal \
+    -c pipeline_client \
+    -m pipeline_model \
+    -t backend/api/go_http_client
+swagger generate client \
+    -f backend/api/swagger/pipeline.upload.swagger.json \
+    -A pipeline_upload \
+    --principal models.Principal \
+    -c pipeline_upload_client \
+    -m pipeline_upload_model \
+    -t backend/api/go_http_client
+swagger generate client \
+    -f backend/api/swagger/visualization.swagger.json \
+    -A visualization \
+    --principal models.Principal \
+    -c visualization_client \
+    -m visualization_model \
+    -t backend/api/go_http_client
+swagger generate client \
+  -f backend/api/swagger/healthz.swagger.json \
+  -A healthz \
+  --principal models.Principal \
+  -c healthz_client \
+  -m healthz_model \
+  -t backend/api/go_http_client
+# Hack to fix an issue with go-swagger
+# See https://github.com/go-swagger/go-swagger/issues/1381 for details.
+sed -i -- 's/MaxConcurrency int64 `json:"max_concurrency,omitempty"`/MaxConcurrency int64 `json:"max_concurrency,omitempty,string"`/g' backend/api/go_http_client/job_model/api_job.go
+sed -i -- 's/IntervalSecond int64 `json:"interval_second,omitempty"`/IntervalSecond int64 `json:"interval_second,omitempty,string"`/g' backend/api/go_http_client/job_model/api_periodic_schedule.go
+sed -i -- 's/MaxConcurrency string `json:"max_concurrency,omitempty"`/MaxConcurrency int64 `json:"max_concurrency,omitempty,string"`/g' backend/api/go_http_client/job_model/api_job.go
+sed -i -- 's/IntervalSecond string `json:"interval_second,omitempty"`/IntervalSecond int64 `json:"interval_second,omitempty,string"`/g' backend/api/go_http_client/job_model/api_periodic_schedule.go
+# Execute the //go:generate directives in the generated code.
+cd backend/api && go generate ./...
+cd ../..