feat(api): image building process improvement (#404)

* feat: adjust kaniko job flags

* feat: implement getHashedModelDependenciesUrl

* feat: create unit test for getHashedModelDependenciesUrl

* feat: refactor app.Dockerfile to use hashed dependency

* feat: call getHashedModelDependenciesUrl and pass to kanikojob

* chore: artifactService dependency injection

* chore: adjust imagebuilder_test

* chore: delete unused attribute and reorder code

* chore: fix appcontext_test

* feat: refactor ensembler job app.Dockerfile

* feat: remove  python installation process on base Dockerfile, install python on the app.Dockerfile instead

* feat: remove python installation from ensembling job base image

* feat: update Makefile to publish turing pyfunc-ensembler-service to pypi

* chore: add version when publishing pyfunc-ensembler-service package

* chore: test app.Dockerfile changes

* chore: try to create new conda env instead of updating the conda env on app.Dockerfile

* chore: change turing_dep value for testing

* chore: change turing_dep value for testing

* feat: install turing-sdk via process_conda_env.sh

* chore: change TURING_DEP value

* feat: use 1 base image for every python version

* chore: remove BaseImageRefTag

* feat: create rule to build pyfunc-ensembler-job setup.py

* feat: adjust app.Dockerfile and base Dockerfile for pyfunc-ensembler-job

* chore: add yq installation step on pyfunc-ensembler-job base Dockerfile

* chore: fix breaking appcontext_test.go

* chore: refactor line with char > 120

* chore: remove unittest related to image tag

* chore: fix golangci-lint related issues

* chore: fix golangci-lint related issues (2)

* chore: fix golangci-lint related issue (3)

* fix: fix wrong arg name in ensembler job app.Dockerfile

* chore: use pyfunc-ensembler-job package instead of dummy package

* feat: add Publish pyfunc-ensembler-job package step on github CI

* chore: adjust github ci for testing

* fix: use importlib.util instead of imp (imp was removed in python 3.12)

* chore: remove TWINE auth from makefile command, use env instead

* chore: adjust pypi related env

* feat: add publish pyfunc-ensembler-service package to github CI

* chore: adjust ci rule for testing

* feat: use importlib.util instead of imp for pyfunc-ensembler-service

* fix: add secret to pyfunc-ensembler-service CI

* chore: remove arg in make build

* feat: add turing prefix to package name

* chore: revert changes related to CI testing

* feat: remove python-version matrix from CI

* chore: separate build and publish rule in Makefile

* feat: use python/ tag to determine pyfunc-ensembler-service and pyfunc-ensembler-job version

* feat: adjust make setup and make test

* chore: adjust github workflow for testing

* chore: remove python version requirement

* fix: fix wrong make setup command

* chore: debug git tag

* chore: remove -y from vertagen

* chore: trigger ci from python/ tag push

* chore: delete Pipfile

* chore: bump up numpy version

* chore: try to bump python version

* chore: fix invalid python version

* feat: use pipenv for make type-check

* chore: fix wrong folder name in make test-type

* chore: add types-PyYAML to dev req

* feat: adjust github CI for pyfunc-ensembler-service

* feat: only run pyfunc-ensembler-* CI for python/ tag release

* feat: change release-rules for sdk, release if new python/ tag is released

* chore: fix make rule called by pyfunc-ensembler-* workflow, add tags trigger for sdk workflow

* fix: change regex to parse tag to dist version

* feat: only run pyfunc-ensembler-* workflow after sdk workflow is successfully run

* feat: remove other trigger for pyfunc-ensembler-* workflow

* fix: change sdk app name param for vertagen

* chore: remove --platform=linux/amd64 from pyfunc-ensembler-job base Dockerfile

* feat: do pyfunc-ensembler-job unit testing for python 3.8 - 3.10

* feat: check error from initArtifactService

* chore: remove BaseImageRef and adjust the unittest

* feat: use artifactService.GetType() instead of artifactServiceType to get artifact service type

* chore: reorder install yq command in Dockerfile

* feat: adjust how to get version from version.py

* feat: remove  from requirements.txt

* feat: use conda instead of pipenv for make setup

* feat: use conda instead of pipenv for make setup

---------

Co-authored-by: Muhammad Naufal Andika Natsir Putra <muhammad.putra@gojek.com>

Author: naufalandika

.github/workflows/pyfunc-ensembler-job.yaml

@@ -1,37 +1,19 @@
 name: engines/pyfunc-ensembler-job
 
 on:
-  # Automatically run CI on Release and Pre-Release tags and main branch
-  # (only if there are changes to relevant paths)
-  push:
-    tags:
-      - "pyfunc-ensembler-job/v[0-9]+.[0-9]+.[0-9]+*"
-    branches:
-      - main
-    paths:
-      - ".github/workflows/pyfunc-ensembler-job.yaml"
-      - "engines/pyfunc-ensembler-job/**"
-      - "sdk/**"
-
-  # Automatically run CI on branches, that have active PR opened
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/pyfunc-ensembler-job.yaml"
-      - "engines/pyfunc-ensembler-job/**"
-      - "sdk/**"
-
-  # To make it possible to trigger e2e CI workflow for any arbitrary git ref
-  workflow_dispatch:
+  # The package build by this job requires latest version of sdk,
+  # so we can only run this workflow after sdk workflow is successfully run
+  workflow_run:
+    workflows: ["sdk"]
+    types:
+      - completed
 
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-    
     steps:
       - uses: actions/checkout@v4
 
@@ -40,7 +22,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache-dependency-path: |
-            engines/pyfunc-ensembler-job/env-${{ matrix.python-version }}.yaml
             engines/pyfunc-ensembler-job/requirements.txt
             engines/pyfunc-ensembler-job/requirements.dev.txt
 
@@ -70,7 +51,7 @@ jobs:
       - id: release-rules
         uses: ./.github/actions/release-rules
         with:
-          prefix: pyfunc-ensembler-job/
+          prefix: python/
 
   publish:
     # Automatically publish release and pre-release artifacts.
@@ -81,14 +62,12 @@ jobs:
     # Dev build can be released either from the 'main' branch or
     # by running this workflow manually with `workflow_dispatch` event.
     if: >-
-      contains('release,pre-release', needs.release-rules.outputs.release-type)
+      (contains('release,pre-release', needs.release-rules.outputs.release-type)
         || ( github.event_name != 'pull_request' )
-        || ( github.event.pull_request.head.repo.full_name == github.repository )
+        || ( github.event.pull_request.head.repo.full_name == github.repository )) &&
+        ${{ github.event.workflow_run.conclusion == 'success' }}
     environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }}
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
     needs:
       - release-rules
       - test
@@ -109,11 +88,20 @@ jobs:
         working-directory: engines/pyfunc-ensembler-job
         env:
           DOCKER_REGISTRY: ghcr.io/${{ github.repository }}
-          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           set -o pipefail
           make build-image | tee output.log
           echo "::set-output name=pyfunc-ensembler-job::$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)"
 
       - name: Publish Pyfunc Ensembler Job Docker Image
         run: docker push ${{ steps.build.outputs.pyfunc-ensembler-job }}
+
+      - name: Publish pyfunc-ensembler-job package
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        working-directory: engines/pyfunc-ensembler-job
+        run: |
+          set -o pipefail
+          make build-and-publish | tee output.log
+          echo "pyfunc-ensembler-job=$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT

.github/workflows/pyfunc-ensembler-service.yaml

@@ -1,46 +1,25 @@
 name: engines/pyfunc-ensembler-service
 
 on:
-  # Automatically run CI on Release and Pre-Release tags and main branch
-  # (only if there are changes to relevant paths)
-  push:
-    tags:
-      - "pyfunc-ensembler-service/v[0-9]+.[0-9]+.[0-9]+*"
-    branches:
-      - main
-    paths:
-      - ".github/workflows/pyfunc-ensembler-service.yaml"
-      - "engines/pyfunc-ensembler-service/**"
-      - "sdk/**"
-
-  # Automatically run CI on branches, that have active PR opened
-  pull_request:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/pyfunc-ensembler-service.yaml"
-      - "engines/pyfunc-ensembler-service/**"
-      - "sdk/**"
-
-  # To make it possible to trigger e2e CI workflow for any arbitrary git ref
-  workflow_dispatch:
+  # The package build by this job requires latest version of sdk,
+  # so we can only run this workflow after sdk workflow is successfully run
+  workflow_run:
+    workflows: ["sdk"]
+    types:
+      - completed
 
 jobs:
   test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup Python ${{ matrix.python-version }}
+      - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.10"
           cache-dependency-path: |
-            engines/pyfunc-ensembler-service/env-${{ matrix.python-version }}.yaml
             engines/pyfunc-ensembler-service/requirements.txt
             engines/pyfunc-ensembler-service/requirements.dev.txt
 
@@ -64,7 +43,7 @@ jobs:
       - id: release-rules
         uses: ./.github/actions/release-rules
         with:
-          prefix: pyfunc-ensembler-service/
+          prefix: python/
 
   publish:
     # Automatically publish release and pre-release artifacts.
@@ -75,14 +54,12 @@ jobs:
     # Dev build can be released either from the 'main' branch or
     # by running this workflow manually with `workflow_dispatch` event.
     if: >-
-      contains('release,pre-release', needs.release-rules.outputs.release-type)
+      (contains('release,pre-release', needs.release-rules.outputs.release-type)
         || ( github.event_name != 'pull_request' )
-        || ( github.event.pull_request.head.repo.full_name == github.repository )
+        || ( github.event.pull_request.head.repo.full_name == github.repository )) &&
+        ${{ github.event.workflow_run.conclusion == 'success' }}
     environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }}
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
     needs:
       - release-rules
       - test
@@ -103,11 +80,20 @@ jobs:
         working-directory: engines/pyfunc-ensembler-service
         env:
           DOCKER_REGISTRY: ghcr.io/${{ github.repository }}
-          PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           set -o pipefail
           make build-image | tee output.log
           echo "::set-output name=pyfunc-ensembler-service-image::$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)"
 
       - name: Publish Pyfunc Ensembler Service Docker Image
         run: docker push ${{ steps.build.outputs.pyfunc-ensembler-service-image }}
+      
+      - name: Publish pyfunc-ensembler-service package
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        working-directory: engines/pyfunc-ensembler-service
+        run: |
+          set -o pipefail
+          make build-and-publish | tee output.log
+          echo "pyfunc-ensembler-service=$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT

.github/workflows/sdk.yaml

@@ -6,6 +6,7 @@ on:
   push:
     tags:
       - "sdk/v[0-9]+.[0-9]+.[0-9]+*"
+      - "python/v[0-9]+.[0-9]+.[0-9]+*"
     branches:
       - main
     paths:
@@ -66,7 +67,7 @@ jobs:
       - id: release-rules
         uses: ./.github/actions/release-rules
         with:
-          prefix: sdk/
+          prefix: python/
 
   publish:
     # Automatically publish release and pre-release artifacts.

api/turing/api/appcontext.go

@@ -10,6 +10,7 @@ import (
 
 	mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster"
 
+	"github.com/caraml-dev/mlp/api/pkg/artifact"
 	batchensembling "github.com/caraml-dev/turing/api/turing/batch/ensembling"
 	batchrunner "github.com/caraml-dev/turing/api/turing/batch/runner"
 	"github.com/caraml-dev/turing/api/turing/cluster"
@@ -94,6 +95,11 @@ func NewAppContext(
 	// Init ensemblers service
 	ensemblersService := service.NewEnsemblersService(db)
 
+	artifactService, err := initArtifactService(cfg)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed initializing artifact service")
+	}
+
 	if cfg.BatchEnsemblingConfig.Enabled {
 		if cfg.BatchEnsemblingConfig.JobConfig == nil {
 			return nil, errors.Wrapf(err, "BatchEnsemblingConfig.JobConfig was not set")
@@ -124,7 +130,7 @@ func NewAppContext(
 		ensemblingImageBuilder, err = imagebuilder.NewEnsemblerJobImageBuilder(
 			imageBuildingController,
 			*cfg.BatchEnsemblingConfig.ImageBuildingConfig,
-			cfg.MlflowConfig.ArtifactServiceType,
+			artifactService,
 		)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Failed initializing ensembling image builder")
@@ -159,7 +165,7 @@ func NewAppContext(
 	ensemblerServiceImageBuilder, err := imagebuilder.NewEnsemblerServiceImageBuilder(
 		clusterControllers[cfg.EnsemblerServiceBuilderConfig.ClusterName],
 		*cfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig,
-		cfg.MlflowConfig.ArtifactServiceType,
+		artifactService,
 	)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Failed initializing ensembler service builder")
@@ -236,3 +242,16 @@ func buildKubeconfigStore(mlpSvc service.MLPService, cfg *config.Config) (map[st
 	}
 	return k8sConfigStore, nil
 }
+
+func initArtifactService(cfg *config.Config) (artifact.Service, error) {
+	if cfg.MlflowConfig.ArtifactServiceType == "gcs" {
+		return artifact.NewGcsArtifactClient()
+	}
+	if cfg.MlflowConfig.ArtifactServiceType == "s3" {
+		return artifact.NewS3ArtifactClient()
+	}
+	if cfg.MlflowConfig.ArtifactServiceType == "nop" {
+		return artifact.NewNopArtifactClient(), nil
+	}
+	return nil, fmt.Errorf("invalid artifact service type %s", cfg.MlflowConfig.ArtifactServiceType)
+}

api/turing/api/appcontext_test.go

@@ -70,10 +70,8 @@ func TestNewAppContext(t *testing.T) {
 				MaxRetryCount:                  3,
 			},
 			ImageBuildingConfig: &config.ImageBuildingConfig{
-				DestinationRegistry: "ghcr.io",
-				BaseImageRef: map[string]string{
-					"3.7.*": "ghcr.io/caraml-dev/turing/pyfunc-ensembler-job:0.0.0-build.1-98b071d",
-				},
+				DestinationRegistry:  "ghcr.io",
+				BaseImage:            "ghcr.io/caraml-dev/turing/pyfunc-ensembler-job:0.0.0-build.1-98b071d",
 				BuildNamespace:       "default",
 				BuildTimeoutDuration: 10 * time.Minute,
 				KanikoConfig: config.KanikoConfig{
@@ -97,10 +95,8 @@ func TestNewAppContext(t *testing.T) {
 		EnsemblerServiceBuilderConfig: config.EnsemblerServiceBuilderConfig{
 			ClusterName: defaultEnvironment,
 			ImageBuildingConfig: &config.ImageBuildingConfig{
-				DestinationRegistry: "ghcr.io",
-				BaseImageRef: map[string]string{
-					"3.7.*": "ghcr.io/caraml-dev/turing/pyfunc-ensembler-service:0.0.0-build.1-98b071d",
-				},
+				DestinationRegistry:  "ghcr.io",
+				BaseImage:            "ghcr.io/caraml-dev/turing/pyfunc-ensembler-service:0.0.0-build.1-98b071d",
 				BuildNamespace:       "default",
 				BuildTimeoutDuration: 10 * time.Minute,
 				KanikoConfig: config.KanikoConfig{
@@ -306,10 +302,13 @@ func TestNewAppContext(t *testing.T) {
 	alertService, err := service.NewGitlabOpsAlertService(nil, *testCfg.AlertConfig)
 	assert.NoError(t, err)
 
+	artifactService, err := initArtifactService(testCfg)
+	assert.NoError(t, err)
+
 	ensemblingImageBuilder, err := imagebuilder.NewEnsemblerJobImageBuilder(
 		nil,
 		*testCfg.BatchEnsemblingConfig.ImageBuildingConfig,
-		testCfg.MlflowConfig.ArtifactServiceType,
+		artifactService,
 	)
 	assert.Nil(t, err)
 
@@ -343,7 +342,7 @@ func TestNewAppContext(t *testing.T) {
 	ensemblerImageBuilder, err := imagebuilder.NewEnsemblerServiceImageBuilder(
 		nil,
 		*testCfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig,
-		testCfg.MlflowConfig.ArtifactServiceType,
+		artifactService,
 	)
 
 	assert.NoError(t, err)

api/turing/batch/ensembling/runner.go

@@ -287,28 +287,9 @@ func (r *ensemblingJobRunner) processOneEnsemblingJob(ensemblingJob *models.Ense
 		return
 	}
 
-	// Get ensembler
-	ensembler, err := r.ensemblersService.FindByID(
-		ensemblingJob.EnsemblerID,
-		service.EnsemblersFindByIDOptions{ProjectID: &ensemblingJob.ProjectID},
-	)
-	if err != nil {
-		r.saveStatusOrTerminate(ensemblingJob, mlpProject, models.JobPending, err.Error(), true)
-		return
-	}
-
-	// Get base image tag
-	var baseImageTag string
-	if pyfuncEnsembler, ok := ensembler.(*models.PyFuncEnsembler); ok {
-		baseImageTag = pyfuncEnsembler.PythonVersion
-	} else {
-		r.saveStatusOrTerminate(ensemblingJob, mlpProject, models.JobPending, err.Error(), true)
-		return
-	}
-
 	// Build Image
 	labels := r.buildLabels(ensemblingJob, mlpProject)
-	imageRef, imageBuildErr := r.buildImage(ensemblingJob, mlpProject, labels, baseImageTag)
+	imageRef, imageBuildErr := r.buildImage(ensemblingJob, mlpProject, labels)
 
 	if imageBuildErr != nil {
 		// Here unfortunately we have to wait till the image building process has
@@ -433,7 +414,6 @@ func (r *ensemblingJobRunner) buildImage(
 	ensemblingJob *models.EnsemblingJob,
 	mlpProject *mlp.Project,
 	buildLabels map[string]string,
-	baseImageTag string,
 ) (string, error) {
 	request := imagebuilder.BuildImageRequest{
 		ProjectName:     mlpProject.Name,
@@ -443,7 +423,6 @@ func (r *ensemblingJobRunner) buildImage(
 		ArtifactURI:     *ensemblingJob.InfraConfig.ArtifactUri,
 		BuildLabels:     buildLabels,
 		EnsemblerFolder: service.EnsemblerFolder,
-		BaseImageRefTag: baseImageTag,
 	}
 	return r.imageBuilder.BuildImage(request)
 }

api/turing/config/config.go

@@ -169,10 +169,6 @@ type ImageBuildingConfig struct {
 	BuildTimeoutDuration time.Duration `validate:"required"`
 	// DestinationRegistry is the registry of the newly built ensembler image.
 	DestinationRegistry string `validate:"required"`
-	// BaseImageRef is the image name of the base ensembler image built from the
-	// engines/pyfunc-ensembler-*/Dockerfile. It's a map of image names, per
-	// minor python version supported by the SDK.
-	BaseImageRef map[string]string `validate:"required"`
 	// KanikoConfig contains the configuration related to the kaniko executor image builder.
 	KanikoConfig KanikoConfig `validate:"required"`
 	// TolerationName allow the scheduler to schedule image building jobs with the matching name
@@ -181,6 +177,8 @@ type ImageBuildingConfig struct {
 	NodeSelector map[string]string
 	// Value for cluster-autoscaler.kubernetes.io/safe-to-evict annotation
 	SafeToEvict bool
+	// BaseImageRef is the image name of the base ensembler image built from the engines/pyfunc-ensembler-*/Dockerfile
+	BaseImage string
 }
 
 // Resource contains the Kubernetes resource request and limits