chore: bump localai v2.18.1 (#317)

.github/workflows/release-base.yaml

@@ -4,9 +4,6 @@ on:
   schedule:
     - cron: "0 0 * * 0"
   workflow_dispatch: # used for testing
-  push:
-    branches:
-      - main
 
 permissions:
   contents: write

.github/workflows/test-docker-args.yaml

@@ -59,8 +59,8 @@ jobs:
             *.blob.core.windows.net:443
             *.azureedge.net:443
             deb.debian.org:80
-            nightly.link:443
             developer.download.nvidia.com:443
+            ghcr.io:443
 
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 

.github/workflows/test-docker-gpu.yaml

@@ -14,7 +14,7 @@ jobs:
       max-parallel: 1
       matrix:
         backend:
-          - llama-cuda
+          # - llama-cuda
           # - exllama
           - exllama2-gptq
           - exllama2-exl2

.github/workflows/test-docker.yaml

@@ -64,9 +64,10 @@ jobs:
             *.blob.core.windows.net:443
             *.azureedge.net:443
             deb.debian.org:80
-            nightly.link:443
             developer.download.nvidia.com:443
             dl-cdn.alpinelinux.org:443
+            *.ubuntu.com:80
+            ghcr.io:443
 
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
@@ -175,7 +176,7 @@ jobs:
       - name: Harden Runner
         uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
         with:
-          egress-policy: audit
+          egress-policy: block
           allowed-endpoints: >
             auth.docker.io:443
             *.huggingface.co:443
@@ -193,11 +194,11 @@ jobs:
             *.blob.core.windows.net:443
             *.azureedge.net:443
             deb.debian.org:80
-            nightly.link:443
             developer.download.nvidia.com:443
             dl-cdn.alpinelinux.org:443
             registry.ollama.ai:443
             *.cloudflarestorage.com:443
+            ghcr.io:443
 
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 

.github/workflows/test-helm.yaml

@@ -58,6 +58,7 @@ jobs:
             *.pkg.dev:443
             *.amazonaws.com:443
             dl-cdn.alpinelinux.org:443
+            ghcr.io:443
 
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 

.github/workflows/test-kubernetes.yaml

@@ -54,6 +54,7 @@ jobs:
             get.helm.sh:443
             *.blob.core.windows.net:443
             *.azureedge.net:443
+            ghcr.io:443
 
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 

Makefile

@@ -27,6 +27,11 @@ build-test-model:
 	docker buildx build . -t ${REGISTRY}/${TEST_IMAGE_NAME}:${TAG} -f ${TEST_FILE} --output=${OUTPUT_TYPE} \
 		--progress=plain --provenance=false
 
+.PHONY: build-distroless-base
+push-distroless-base:
+	docker buildx build . -t sozercan/aikit-base:latest -f Dockerfile.base --platform linux/amd64,linux/arm64 \
+		--sbom=true --push
+
 .PHONY: run-test-model
 run-test-model:
 	docker run --rm -p 8080:8080 ${REGISTRY}/${TEST_IMAGE_NAME}:${TAG}

pkg/aikit2llb/inference/convert.go

@@ -12,17 +12,16 @@ import (
 )
 
 const (
-	distrolessBase = "gcr.io/distroless/cc-debian12:latest"
-
+	distrolessBase = "ghcr.io/sozercan/base:latest"
 	localAIRepo    = "https://github.com/mudler/LocalAI"
-	localAIVersion = "v2.16.0"
+	localAIVersion = "v2.18.1"
 	cudaVersion    = "12-5"
 )
 
 // Aikit2LLB converts an InferenceConfig to an LLB state.
 func Aikit2LLB(c *config.InferenceConfig, platform *specs.Platform) (llb.State, *specs.Image, error) {
 	var merge llb.State
-	state := llb.Image(utils.DebianSlim, llb.Platform(*platform))
+	state := llb.Image(utils.UbuntuBase, llb.Platform(*platform))
 	base := getBaseImage(c, platform)
 
 	var err error
@@ -47,7 +46,7 @@ func Aikit2LLB(c *config.InferenceConfig, platform *specs.Platform) (llb.State,
 		case utils.BackendExllama, utils.BackendExllamaV2:
 			merge = installExllama(c, state, merge)
 		case utils.BackendStableDiffusion:
-			merge = installOpenCV(state, merge, *platform)
+			merge = installOpenCV(state, merge)
 		case utils.BackendMamba:
 			merge = installMamba(state, merge)
 		}
@@ -60,7 +59,7 @@ func Aikit2LLB(c *config.InferenceConfig, platform *specs.Platform) (llb.State,
 // getBaseImage returns the base image given the InferenceConfig and platform.
 func getBaseImage(c *config.InferenceConfig, platform *specs.Platform) llb.State {
 	if len(c.Backends) > 0 {
-		return llb.Image(utils.DebianSlim, llb.Platform(*platform))
+		return llb.Image(utils.UbuntuBase, llb.Platform(*platform))
 	}
 	return llb.Image(distrolessBase, llb.Platform(*platform))
 }
@@ -109,7 +108,7 @@ func copyModels(c *config.InferenceConfig, base llb.State, s llb.State, platform
 
 // installCuda installs cuda libraries and dependencies.
 func installCuda(c *config.InferenceConfig, s llb.State, merge llb.State) (llb.State, llb.State) {
-	cudaKeyringURL := "https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb"
+	cudaKeyringURL := "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb"
 	cudaKeyring := llb.HTTP(cudaKeyringURL)
 	s = s.File(
 		llb.Copy(cudaKeyring, utils.FileNameFromURL(cudaKeyringURL), "/"),
@@ -125,10 +124,7 @@ func installCuda(c *config.InferenceConfig, s llb.State, merge llb.State) (llb.S
 	if len(c.Backends) == 0 {
 		// install cuda libraries and pciutils for gpu detection
 		s = s.Run(utils.Shf("apt-get install -y --no-install-recommends pciutils libcublas-%[1]s cuda-cudart-%[1]s && apt-get clean", cudaVersion)).Root()
-		// using a distroless base image here
-		// convert debian package metadata status file to distroless status.d directory
-		// clean up apt directories
-		s = s.Run(utils.Bashf("apt-get install -y --no-install-recommends libcublas-%[1]s cuda-cudart-%[1]s && apt-get clean && mkdir -p /var/lib/dpkg/status.d && description_flag=false; while IFS= read -r line || [[ -n $line ]]; do if [[ $line == Package:* ]]; then pkg_name=$(echo $line | cut -d' ' -f2); elif [[ $line == Maintainer:* ]]; then maintainer=$(echo $line | cut -d' ' -f2-); if [[ $maintainer == 'cudatools <cudatools@nvidia.com>' ]]; then pkg_file=/var/lib/dpkg/status.d/${pkg_name}; echo 'Package: '$pkg_name > $pkg_file; echo $line >> $pkg_file; else pkg_file=''; fi; elif [[ -n $pkg_file ]]; then if [[ $line == Description:* ]]; then description_flag=true; elif [[ $line == '' ]]; then description_flag=false; elif ! $description_flag; then echo $line >> $pkg_file; fi; fi; done < /var/lib/dpkg/status && find /var/lib/dpkg -mindepth 1 ! -regex '^/var/lib/dpkg/status\\.d\\(/.*\\)?' -delete && rm -r /var/lib/apt", cudaVersion)).Root()
+		// TODO: clean up /var/lib/dpkg/status
 	}
 
 	// installing dev dependencies used for exllama
@@ -163,8 +159,7 @@ func addLocalAI(s llb.State, merge llb.State, platform specs.Platform) (llb.Stat
 	if !exists {
 		return s, merge, fmt.Errorf("unsupported architecture %s", platform.Architecture)
 	}
-	// TODO: update this URL when the binary is available in github
-	localAIURL := fmt.Sprintf("https://sertaccdnvs.azureedge.net/localai/%[1]s/%[2]s", localAIVersion, binaryName)
+	localAIURL := fmt.Sprintf("https://github.com/mudler/LocalAI/releases/download/%[1]s/%[2]s", localAIVersion, binaryName)
 
 	savedState := s
 

pkg/aikit2llb/inference/exllama.go

@@ -15,7 +15,7 @@ func installExllama(c *config.InferenceConfig, s llb.State, merge llb.State) llb
 	}
 
 	savedState := s
-	s = s.Run(utils.Sh("apt-get update && apt-get install --no-install-recommends -y bash git ca-certificates python3-pip python3-dev python3-venv python-is-python3 make g++ curl && pip install uv grpcio-tools --break-system-packages && apt-get clean"), llb.IgnoreCache).Root()
+	s = s.Run(utils.Sh("apt-get update && apt-get install --no-install-recommends -y bash git ca-certificates python3-pip python3-dev python3-venv python-is-python3 make g++ curl && pip install uv grpcio-tools && apt-get clean"), llb.IgnoreCache).Root()
 
 	s = cloneLocalAI(s)
 

pkg/aikit2llb/inference/mamba.go

@@ -8,7 +8,7 @@ import (
 func installMamba(s llb.State, merge llb.State) llb.State {
 	savedState := s
 	// libexpat1 is requirement but git is not. however libexpat1 is a dependency of git
-	s = s.Run(utils.Sh("apt-get install --no-install-recommends -y git python3 python3-dev python3-pip python3-venv python-is-python3 libssl3 openssl curl && pip install uv grpcio-tools --break-system-packages && apt-get clean"), llb.IgnoreCache).Root()
+	s = s.Run(utils.Sh("apt-get install --no-install-recommends -y git python3 python3-dev python3-pip python3-venv python-is-python3 libssl3 openssl curl && pip install uv grpcio-tools && apt-get clean"), llb.IgnoreCache).Root()
 
 	s = cloneLocalAI(s)
 

pkg/aikit2llb/inference/stablediffusion.go

@@ -4,31 +4,16 @@ import (
 	"fmt"
 
 	"github.com/moby/buildkit/client/llb"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sozercan/aikit/pkg/utils"
 )
 
-func installOpenCV(s llb.State, merge llb.State, platform specs.Platform) llb.State {
-	// pinning libdap packages to bullseye version due to symbol error
-	libdapVersion := "3.20.7-6"
-	libPaths := map[string]string{
-		utils.PlatformAMD64: "/usr/lib/x86_64-linux-gnu",
-		utils.PlatformARM64: "/usr/lib/aarch64-linux-gnu",
-	}
-	libPath, exists := libPaths[platform.Architecture]
-	if !exists {
-		return s
-	}
-
+func installOpenCV(s llb.State, merge llb.State) llb.State {
 	savedState := s
-	// adding debian 11 (bullseye) repo due to opencv 4.5 requirement
-	s = s.Run(utils.Sh("echo 'deb http://deb.debian.org/debian bullseye main' | tee -a /etc/apt/sources.list")).Root()
-	s = s.Run(utils.Shf("apt-get update && mkdir -p /tmp/generated/images && apt-get install --no-install-recommends -y curl unzip ca-certificates libopencv-imgcodecs4.5 libgomp1 libdap27=%[1]s libdapclient6v5=%[1]s && apt-get clean && ln -s %[2]s/libopencv_core.so.4.5 %[2]s/libopencv_core.so.4.5d && ln -s %[2]s/libopencv_core.so.4.5 %[2]s/libopencv_core.so.406 && ln -s %[2]s/libopencv_imgcodecs.so.4.5 %[2]s/libopencv_imgcodecs.so.4.5d", libdapVersion, libPath), llb.IgnoreCache).Root()
+	s = s.Run(utils.Sh("apt-get update && mkdir -p /tmp/generated/images && apt-get install --no-install-recommends -y libopencv-imgcodecs4.5d && apt-get clean"), llb.IgnoreCache).Root()
 	diff := llb.Diff(savedState, s)
 	merge = llb.Merge([]llb.State{merge, diff})
 
-	// TODO: update this URL when the binary is available in github
-	sdURL := fmt.Sprintf("https://sertaccdnvs.azureedge.net/localai/%s/stablediffusion", localAIVersion)
+	sdURL := fmt.Sprintf("https://github.com/mudler/LocalAI/releases/download/%[1]s/stablediffusion", localAIVersion)
 	var opts []llb.HTTPOption
 	opts = append(opts, llb.Filename("stablediffusion"), llb.Chmod(0o755))
 	var copyOpts []llb.CopyOption

pkg/utils/const.go

@@ -17,7 +17,7 @@ const (
 
 	APIv1alpha1 = "v1alpha1"
 
-	DebianSlim = "docker.io/library/debian:12-slim"
+	UbuntuBase = "docker.io/library/ubuntu:22.04"
 	CudaDevel  = "nvcr.io/nvidia/cuda:12.3.2-devel-ubuntu22.04"
 
 	PlatformLinux = "linux"

test/aikitfile-dev.yaml

@@ -1,4 +1,4 @@
-#syntax=ghcr.io/sozercan/aikit/inference:dev
+#syntax=ghcr.io/sozercan/aikit:dev
 apiVersion: v1alpha1
 debug: true
 models: