Add layer "pull" support via generic code archive download

cmd/localstack/awsutil.go

@@ -6,7 +6,6 @@
 package main
 
 import (
-	"archive/zip"
 	"context"
 	"fmt"
 	"github.com/jessevdk/go-flags"
@@ -18,7 +17,6 @@ import (
 	"math"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -27,7 +25,6 @@ import (
 const (
 	optBootstrap     = "/opt/bootstrap"
 	runtimeBootstrap = "/var/runtime/bootstrap"
-	taskFolder       = "/var/task"
 )
 
 type options struct {
@@ -134,70 +131,6 @@ func GetenvWithDefault(key string, defaultValue string) string {
 	return envValue
 }
 
-func DownloadCodeArchive(url string) {
-	// download and unzip code archive, if url is given
-	if url == "" {
-		return
-	}
-	log.Infoln("Downloading code archive")
-	// create tmp directory
-	tmpDir := os.TempDir()
-	// download code archive into tmp directory
-	res, err := http.Get(url)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer res.Body.Close()
-	tmp_file_path := path.Join(tmpDir, "code-archive.zip")
-	tmp_file, err := os.OpenFile(tmp_file_path, os.O_WRONLY|os.O_CREATE, os.ModePerm)
-	if err != nil {
-		log.Fatal(err)
-	}
-	_, err = io.Copy(tmp_file, res.Body)
-	if err != nil {
-		log.Fatal(err)
-	}
-	err = tmp_file.Close()
-	if err != nil {
-		log.Fatal(err)
-	}
-	// unzip into /var/task
-	log.Infoln("Unzipping code archive")
-	r, err := zip.OpenReader(tmp_file_path)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer r.Close()
-	for _, f := range r.File {
-		rc, err := f.Open()
-		if err != nil {
-			log.Fatal(err)
-		}
-		target_file_name := path.Join(taskFolder, f.Name)
-		if f.FileInfo().IsDir() {
-			err = os.MkdirAll(target_file_name, os.ModePerm)
-			if err != nil {
-				log.Fatal(err)
-			}
-			continue
-		}
-		if err := os.MkdirAll(filepath.Dir(target_file_name), os.ModePerm); err != nil {
-			panic(err)
-		}
-		target_file, err := os.OpenFile(target_file_name, os.O_WRONLY|os.O_CREATE, os.ModePerm)
-		if err != nil {
-			log.Fatal(err)
-		}
-		_, err = io.Copy(target_file, rc)
-		if err != nil {
-			log.Fatal(err)
-		}
-		target_file.Close()
-		rc.Close()
-	}
-
-}
-
 func resetListener(changeChannel <-chan bool, server *CustomInteropServer) {
 	for {
 		_, more := <-changeChannel

cmd/localstack/codearchive.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"archive/zip"
+	"encoding/json"
+	log "github.com/sirupsen/logrus"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+type ArchiveDownload struct {
+	Url        string `json:"url"`
+	TargetPath string `json:"target_path"`
+}
+
+func DownloadCodeArchives(archives string) error {
+	if archives == "" {
+		log.Debugln("No code archives set. Skipping download.")
+		return nil
+	}
+	var parsedArchives []ArchiveDownload
+	err := json.Unmarshal([]byte(archives), &parsedArchives)
+	if err != nil {
+		return err
+	}
+
+	for _, downloadArchive := range parsedArchives {
+		if err := DownloadCodeArchive(downloadArchive.Url, downloadArchive.TargetPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func DownloadCodeArchive(url string, targetPath string) error {
+	// download and unzip code archive
+	log.Infoln("Downloading code archive")
+	// create tmp directory
+	// empty string will make use of the default tmp directory
+	tmpDir, err := os.MkdirTemp("", "localstack-code-archive")
+	if err != nil {
+		return err
+	}
+	// download code archive into tmp directory
+	res, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	tmp_file_path := path.Join(tmpDir, "code-archive.zip")
+	tmp_file, err := os.OpenFile(tmp_file_path, os.O_WRONLY|os.O_CREATE, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(tmp_file, res.Body)
+	if err != nil {
+		return err
+	}
+	err = tmp_file.Close()
+	if err != nil {
+		return err
+	}
+	// unzip into targetPath
+	log.Infoln("Unzipping code archive")
+	r, err := zip.OpenReader(tmp_file_path)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	for _, f := range r.File {
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		// TODO: check if exists, otherwise build path
+		target_file_name := path.Join(targetPath, f.Name)
+		if f.FileInfo().IsDir() {
+			err = os.MkdirAll(target_file_name, os.ModePerm)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		if err := os.MkdirAll(filepath.Dir(target_file_name), os.ModePerm); err != nil {
+			panic(err)
+		}
+		target_file, err := os.OpenFile(target_file_name, os.O_WRONLY|os.O_CREATE, os.ModePerm)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(target_file, rc)
+		if err != nil {
+			return err
+		}
+		target_file.Close()
+		rc.Close()
+	}
+	return nil
+}

cmd/localstack/main.go

@@ -19,7 +19,7 @@ type LsOpts struct {
 	RuntimeEndpoint   string
 	RuntimeId         string
 	InitTracingPort   string
-	CodeDownloadUrl   string
+	CodeArchives      string
 	HotReloadingPaths []string
 	EnableDnsServer   string
 	LocalstackIP      string
@@ -41,7 +41,7 @@ func InitLsOpts() *LsOpts {
 		InteropPort:     GetenvWithDefault("LOCALSTACK_INTEROP_PORT", "9563"),
 		InitTracingPort: GetenvWithDefault("LOCALSTACK_RUNTIME_TRACING_PORT", "9564"),
 		// optional or empty
-		CodeDownloadUrl:   os.Getenv("LOCALSTACK_CODE_ARCHIVE_DOWNLOAD_URL"),
+		CodeArchives:      os.Getenv("LOCALSTACK_CODE_ARCHIVES"),
 		HotReloadingPaths: strings.Split(GetenvWithDefault("LOCALSTACK_HOT_RELOADING_PATHS", ""), ","),
 		EnableDnsServer:   os.Getenv("LOCALSTACK_ENABLE_DNS_SERVER"),
 		LocalstackIP:      os.Getenv("LOCALSTACK_HOSTNAME"),
@@ -59,8 +59,11 @@ func main() {
 	//log.SetLevel(log.TraceLevel)
 	log.SetLevel(log.DebugLevel)
 	log.SetReportCaller(true)
+
 	// download code archive if env variable is set
-	DownloadCodeArchive(lsOpts.CodeDownloadUrl)
+	if err := DownloadCodeArchives(lsOpts.CodeArchives); err != nil {
+		log.Fatal("Failed to download code archives")
+	}
 	// enable dns server
 	dnsServerContext, stopDnsServer := context.WithCancel(context.Background())
 	go RunDNSRewriter(lsOpts, dnsServerContext)