Code restructuring to support V1alpha1 and V1alpha2 API

Makefile

@@ -0,0 +1,26 @@
+# Run tests
+test:
+	go test ./pkg/... ./cmd/...
+
+# Build Katib images
+build: 
+	sh scripts/build.sh
+
+# Deploy katib manifests into a k8s cluster
+deploy: 
+	sh scripts/deploy.sh
+
+# Run go fmt against code
+fmt:
+	go fmt ./pkg/... ./cmd/...
+
+# Run go vet against code
+vet:
+	go vet ./pkg/... ./cmd/...
+
+# Generate code
+generate:
+ifndef GOPATH
+	$(error GOPATH not defined, please define GOPATH. Run "go help gopath" to learn more about GOPATH)
+endif
+	go generate ./pkg/... ./cmd/...

cmd/earlystopping/medianstopping/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:alpine AS build-env
 # The GOPATH in the image is /go.
 ADD . /go/src/github.com/kubeflow/katib
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/earlystopping/medianstopping
-RUN go build -o medianstopping
+RUN go build -o medianstopping ./v1alpha1
 
 FROM alpine:3.7
 WORKDIR /app

cmd/earlystopping/medianstopping/main.go → cmd/earlystopping/medianstopping/v1alpha1/main.go

@@ -7,8 +7,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 
-	pb "github.com/kubeflow/katib/pkg/api"
-	"github.com/kubeflow/katib/pkg/earlystopping"
+	pb "github.com/kubeflow/katib/pkg/api/v1alpha1"
+	earlystopping "github.com/kubeflow/katib/pkg/earlystopping/v1alpha1"
 )
 
 func main() {

cmd/manager-rest/Dockerfile

@@ -2,9 +2,11 @@ FROM golang:alpine AS build-env
 # The GOPATH in the image is /go.
 ADD . /go/src/github.com/kubeflow/katib
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/manager-rest
-RUN go build -o vizier-manager-rest
+RUN go build -o vizier-manager-rest ./v1alpha1
+RUN go build -o katib-manager-rest.v1alpha2 ./v1alpha2
 
 FROM alpine:3.7
 WORKDIR /app
 COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager-rest/vizier-manager-rest /app/
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager-rest/katib-manager-rest.v1alpha2 /app/
 ENTRYPOINT ["./vizier-manager-rest"]

cmd/manager-rest/v1alpha1/proxy.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/golang/glog"
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	gw "github.com/kubeflow/katib/pkg/api/v1alpha1"
+)
+
+var (
+	vizierCoreEndpoint = flag.String("echo_endpoint", "vizier-core:6789", "vizier-core endpoint")
+)
+
+func run() error {
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	mux := runtime.NewServeMux()
+	opts := []grpc.DialOption{grpc.WithInsecure()}
+	// register handlers for the HTTP endpoints
+	err := gw.RegisterManagerHandlerFromEndpoint(ctx, mux, *vizierCoreEndpoint, opts)
+	if err != nil {
+		return err
+	}
+
+	// proxy server listens on port 80
+	return http.ListenAndServe(":80", mux)
+}
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	if err := run(); err != nil {
+		glog.Fatal(err)
+	}
+}

cmd/manager-rest/v1alpha2/proxy.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/golang/glog"
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	gw "github.com/kubeflow/katib/pkg/api/v1alpha2"
+)
+
+var (
+	vizierCoreEndpoint = flag.String("echo_endpoint", "vizier-core:6789", "vizier-core endpoint")
+)
+
+func run() error {
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	mux := runtime.NewServeMux()
+	opts := []grpc.DialOption{grpc.WithInsecure()}
+	// register handlers for the HTTP endpoints
+	err := gw.RegisterManagerHandlerFromEndpoint(ctx, mux, *vizierCoreEndpoint, opts)
+	if err != nil {
+		return err
+	}
+
+	// proxy server listens on port 80
+	return http.ListenAndServe(":80", mux)
+}
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	if err := run(); err != nil {
+		glog.Fatal(err)
+	}
+}

cmd/manager/Dockerfile

@@ -2,14 +2,16 @@ FROM golang:alpine AS build-env
 # The GOPATH in the image is /go.
 ADD . /go/src/github.com/kubeflow/katib
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/manager
-RUN go build -o vizier-manager
+RUN go build -o vizier-manager ./v1alpha1
+RUN go build -o katib-manager.v1alpha2 ./v1alpha2
 
 FROM alpine:3.7
 WORKDIR /app
 RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
     chmod +x /bin/grpc_health_probe
 COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager/vizier-manager /app/
-COPY --from=build-env /go/src/github.com/kubeflow/katib/pkg/manager/visualise /
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager/katib-manager.v1alpha2 /app/
+#COPY --from=build-env /go/src/github.com/kubeflow/katib/pkg/manager/v1alpha1/visualise /
 ENTRYPOINT ["./vizier-manager"]
 CMD ["-w", "kubernetes"]

cmd/manager/main.go → cmd/manager/v1alpha1/main.go

@@ -9,9 +9,9 @@ import (
 	"net"
 	"time"
 
-	api_pb "github.com/kubeflow/katib/pkg/api"
-	health_pb "github.com/kubeflow/katib/pkg/api/health"
-	kdb "github.com/kubeflow/katib/pkg/db"
+	api_pb "github.com/kubeflow/katib/pkg/api/v1alpha1"
+	health_pb "github.com/kubeflow/katib/pkg/api/v1alpha1/health"
+	kdb "github.com/kubeflow/katib/pkg/db/v1alpha1"
 	"github.com/kubeflow/katib/pkg/manager/modelstore"
 
 	"google.golang.org/grpc"

cmd/manager/main_test.go → cmd/manager/v1alpha1/main_test.go

@@ -6,8 +6,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 
-	api "github.com/kubeflow/katib/pkg/api"
-	mockdb "github.com/kubeflow/katib/pkg/mock/db"
+	api "github.com/kubeflow/katib/pkg/api/v1alpha1"
+	mockdb "github.com/kubeflow/katib/pkg/mock/v1alpha1/db"
 )
 
 func TestCreateStudy(t *testing.T) {

cmd/manager/v1alpha2/main.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"net"
+
+	api_pb "github.com/kubeflow/katib/pkg/api/v1alpha2"
+	health_pb "github.com/kubeflow/katib/pkg/api/v1alpha2/health"
+	kdb "github.com/kubeflow/katib/pkg/db/v1alpha2"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/reflection"
+)
+
+const (
+	port = "0.0.0.0:6789"
+)
+
+var dbIf kdb.KatibDBInterface
+
+type server struct {
+}
+
+// Register a Experiment to DB.
+func (s *server) RegisterExperiment(context.Context, *api_pb.RegisterExperimentRequest) (*api_pb.RegisterExperimentReply, error) {
+	return nil, nil
+}
+
+// Delete a Experiment from DB by name.
+func (s *server) DeleteExperiment(context.Context, *api_pb.DeleteExperimentRequest) (*api_pb.DeleteExperimentReply, error) {
+	return nil, nil
+}
+
+// Get a Experiment from DB by name.
+func (s *server) GetExperiment(context.Context, *api_pb.GetExperimentRequest) (*api_pb.GetExperimentReply, error) {
+	return nil, nil
+}
+
+// Get a summary list of Experiment from DB.
+// The summary includes name and condition.
+func (s *server) GetExperimentList(context.Context, *api_pb.GetExperimentListRequest) (*api_pb.GetExperimentListReply, error) {
+	return nil, nil
+}
+
+// Update Status of a experiment.
+func (s *server) UpdateExperimentStatus(context.Context, *api_pb.UpdateExperimentStatusRequest) (*api_pb.UpdateExperimentStatusReply, error) {
+	return nil, nil
+}
+
+// Update AlgorithmExtraSettings.
+// The ExtraSetting is created if it does not exist, otherwise it is overwrited.
+func (s *server) UpdateAlgorithmExtraSettings(context.Context, *api_pb.UpdateAlgorithmExtraSettingsRequest) (*api_pb.UpdateAlgorithmExtraSettingsReply, error) {
+	return nil, nil
+}
+
+// Get all AlgorithmExtraSettings.
+func (s *server) GetAlgorithmExtraSettings(context.Context, *api_pb.GetAlgorithmExtraSettingsRequest) (*api_pb.GetAlgorithmExtraSettingsReply, error) {
+	return nil, nil
+}
+
+// Register a Trial to DB.
+// ID will be filled by manager automatically.
+func (s *server) RegisterTrial(context.Context, *api_pb.RegisterTrialRequest) (*api_pb.RegisterTrialReply, error) {
+	return nil, nil
+}
+
+// Delete a Trial from DB by ID.
+func (s *server) DeleteTrial(context.Context, *api_pb.DeleteTrialRequest) (*api_pb.DeleteTrialReply, error) {
+	return nil, nil
+}
+
+// Get a list of Trial from DB by name of a Experiment.
+func (s *server) GetTrialList(context.Context, *api_pb.GetTrialListRequest) (*api_pb.GetTrialListReply, error) {
+	return nil, nil
+}
+
+// Get a Trial from DB by ID of Trial.
+func (s *server) GetTrial(context.Context, *api_pb.GetTrialRequest) (*api_pb.GetTrialReply, error) {
+	return nil, nil
+}
+
+// Update Status of a trial.
+func (s *server) UpdateTrialStatus(context.Context, *api_pb.UpdateTrialStatusRequest) (*api_pb.UpdateTrialStatusReply, error) {
+	return nil, nil
+}
+
+// Report a log of Observations for a Trial.
+// The log consists of timestamp and value of metric.
+// Katib store every log of metrics.
+// You can see accuracy curve or other metric logs on UI.
+func (s *server) ReportObservationLog(context.Context, *api_pb.ReportObservationLogRequest) (*api_pb.ReportObservationLogReply, error) {
+	return nil, nil
+}
+
+// Get all log of Observations for a Trial.
+func (s *server) GetObservationLog(context.Context, *api_pb.GetObservationLogRequest) (*api_pb.GetObservationLogReply, error) {
+	return nil, nil
+}
+
+// Get Suggestions from a Suggestion service.
+func (s *server) GetSuggestions(context.Context, *api_pb.GetSuggestionsRequest) (*api_pb.GetSuggestionsReply, error) {
+	return nil, nil
+}
+
+// Validate AlgorithmSettings in an Experiment.
+// Suggestion service should return INVALID_ARGUMENT Error when the parameter is invalid
+func (s *server) ValidateAlgorithmSettings(context.Context, *api_pb.ValidateAlgorithmSettingsRequest) (*api_pb.ValidateAlgorithmSettingsReply, error) {
+	return nil, nil
+}
+
+func (s *server) Check(context.Context, *health_pb.HealthCheckRequest) (*health_pb.HealthCheckResponse, error) {
+	return nil, nil
+}
+
+func main() {
+	flag.Parse()
+	var err error
+
+	dbIf, err = kdb.New()
+	if err != nil {
+		log.Fatalf("Failed to open db connection: %v", err)
+	}
+	dbIf.DBInit()
+	listener, err := net.Listen("tcp", port)
+	if err != nil {
+		log.Fatalf("Failed to listen: %v", err)
+	}
+
+	size := 1<<31 - 1
+	log.Printf("Start Katib manager: %s", port)
+	s := grpc.NewServer(grpc.MaxRecvMsgSize(size), grpc.MaxSendMsgSize(size))
+	api_pb.RegisterManagerServer(s, &server{})
+	health_pb.RegisterHealthServer(s, &server{})
+	reflection.Register(s)
+	if err = s.Serve(listener); err != nil {
+		log.Fatalf("Failed to serve: %v", err)
+	}
+}

cmd/metricscollector/Dockerfile

@@ -6,7 +6,7 @@ ADD . /go/src/github.com/kubeflow/katib
 
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/metricscollector
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o metricscollector
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o metricscollector ./v1alpha1
 # Copy the controller-manager into a thin image
 FROM alpine:3.7
 WORKDIR /app