diff --git a/build.sh b/build.sh index aa15385..95708ee 100755 --- a/build.sh +++ b/build.sh @@ -16,6 +16,9 @@ root_dir=$(cd "$(dirname "$0")"; echo "$PWD") echo "ROOT DIR: ${root_dir}" cd "${root_dir}" +# Disabling cgo in order to not link with libc and utilize static linkage binaries +# which will help to not relay on glibc on linux and be truely independend from OS +export CGO_ENABLED=0 echo "--- GENERATE CODE FOR AQUARIUM-FISH ---" # Install oapi-codegen if it's not available or version is not the same with go.mod @@ -66,7 +69,8 @@ fi # Run parallel builds but no more than limit (gox doesn't support all the os/archs we need) pwait() { # Note: Dash really don't like jobs to be executed in a pipe or in other shell, soooo... - while jobs > /tmp/jobs_list.tmp; do + # Using "-p" to show only PIDs (because it could be multiline) and "-r" to show only running jobs + while jobs -pr > /tmp/jobs_list.tmp; do [ $(cat /tmp/jobs_list.tmp | wc -l) -ge $1 ] || break sleep 1 done @@ -117,29 +121,31 @@ done [ $errorcount -eq 0 ] || exit $errorcount -echo -echo "--- ARCHIVE ${BINARY_NAME} ($MAXJOBS in parallel) ---" - -# Pack the artifact archives -for GOOS in $os_list; do - for GOARCH in $arch_list; do - name="$BINARY_NAME.${GOOS}_${GOARCH}" - [ -f "$name" ] || continue - - echo "Archiving: $name ..." - mkdir "$name.dir" - bin_name='aquarium-fish' - [ "$GOOS" != "windows" ] || bin_name="$bin_name.exe" - - cp -a "$name" "$name.dir/$bin_name" - $( - cd "$name.dir" - tar -cJf "../$name.tar.xz" "$bin_name" >/dev/null 2>&1 - zip "../$name.zip" "$bin_name" >/dev/null 2>&1 - cd .. && rm -rf "$name.dir" - ) & - pwait $MAXJOBS +if [ "x${RELEASE}" != "x" ]; then + echo + echo "--- ARCHIVE ${BINARY_NAME} ($MAXJOBS in parallel) ---" + + # Pack the artifact archives + for GOOS in $os_list; do + for GOARCH in $arch_list; do + name="$BINARY_NAME.${GOOS}_${GOARCH}" + [ -f "$name" ] || continue + + echo "Archiving: $(du -h "$name") ..." + mkdir "$name.dir" + bin_name='aquarium-fish' + [ "$GOOS" != "windows" ] || bin_name="$bin_name.exe" + + cp -a "$name" "$name.dir/$bin_name" + $( + cd "$name.dir" + tar -cJf "../$name.tar.xz" "$bin_name" >/dev/null 2>&1 + zip "../$name.zip" "$bin_name" >/dev/null 2>&1 + cd .. && rm -rf "$name.dir" + ) & + pwait $MAXJOBS + done done -done -wait + wait +fi diff --git a/check.sh b/check.sh index 4a22bca..286d8f1 100755 --- a/check.sh +++ b/check.sh @@ -32,7 +32,7 @@ done echo echo '---------------------- GoFmt verify ----------------------' echo -reformat=$(gofmt -l .) +reformat=$(gofmt -l . 2>&1) if [ "${reformat}" ]; then echo "Please run 'gofmt -w .': \n${reformat}" errors=$((${errors}+$(echo "${reformat}" | wc -l))) @@ -57,7 +57,7 @@ echo vet=$(go vet ./... 2>&1) if [ "${vet}" ]; then echo "Please fix the issues: \n${vet}" - errors=$(((${errors}+$(echo "${vet}" | wc -l))/2)) + errors=$(( ${errors}+$(echo "${vet}" | wc -l) )) fi diff --git a/cmd/fish/fish.go b/cmd/fish/fish.go index ba9d02c..8afe1a3 100644 --- a/cmd/fish/fish.go +++ b/cmd/fish/fish.go @@ -33,6 +33,8 @@ import ( ) func main() { + log.Infof("Aquarium Fish %s (%s)", build.Version, build.Time) + var api_address string var proxy_address string var node_address string @@ -56,8 +58,6 @@ func main() { return log.InitLoggers() }, RunE: func(cmd *cobra.Command, args []string) error { - log.Infof("Aquarium Fish %s (%s)", build.Version, build.Time) - cfg := &fish.Config{} if err := cfg.ReadConfigFile(cfg_path); err != nil { return log.Error("Fish: Unable to apply config file:", cfg_path, err) diff --git a/go.mod b/go.mod index b3483ed..07b4d2d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/adobe/aquarium-fish -go 1.20 +go 1.21 require ( github.com/alessio/shellescape v1.4.1 diff --git a/go.sum b/go.sum index f81769d..3d6c87c 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= diff --git a/lib/drivers/native/config.go b/lib/drivers/native/config.go index 24ec6e9..6aab020 100644 --- a/lib/drivers/native/config.go +++ b/lib/drivers/native/config.go @@ -28,10 +28,24 @@ import ( type Config struct { //TODO: Users []string `json:"users"` // List of precreated OS user names in format "user[:password]" to run the workload - SudoPath string `json:"sudo_path"` // Path to the sudo (privilege escalation) binary + SuPath string `json:"su_path"` // Path to the su (login as user) binary + SudoPath string `json:"sudo_path"` // Path to the sudo (privilege escalation) binary + ShPath string `json:"sh_path"` // Path to the sh (simple user shell) binary + TarPath string `json:"tar_path"` // Path to the tar (unpacking images) binary + MountPath string `json:"mount_path"` // Path to the mount (list of mounted volumes) binary + ChownPath string `json:"chown_path"` // Path to the chown (change file/dir ownership) binary + ChmodPath string `json:"chmod_path"` // Path to the chmod (change file/dir access) binary + KillallPath string `json:"killall_path"` // Path to the killall (send signals to multiple processes) binary + RmPath string `json:"rm_path"` // Path to the rm (cleanup after execution) binary + ImagesPath string `json:"images_path"` // Where to store/look the environment images WorkspacePath string `json:"workspace_path"` // Where to place the env disks + DsclPath string `json:"dscl_path"` // Path to the dscl (macos user control) binary + HdiutilPath string `json:"hdiutil_path"` // Path to the hdiutil (macos images create/mount/umount) binary + MdutilPath string `json:"mdutil_path"` // Path to the mdutil (macos disable indexing for disks) binary + CreatehomedirPath string `json:"createhomedir_path"` // Path to the createhomedir (macos create/prefill user directory) binary + // Alter allows you to control how much resources will be used: // * Negative (<0) value will alter the total resource count before provisioning so you will be // able to save some resources for the host system (recommended -2 for CPU and -10 for RAM @@ -89,8 +103,8 @@ func (c *Config) Apply(config []byte) (err error) { } func (c *Config) Validate() (err error) { - // Sudo is used to become the separated unprevileged user which will execute the workload - // and execute a number of administrative actions to create/delete the user and cleanup + // Sudo is used to run commands from superuser and execute a number of + // administrative actions to create/delete the user and cleanup if c.SudoPath == "" { if c.SudoPath, err = exec.LookPath("sudo"); err != nil { return fmt.Errorf("Native: Unable to locate `sudo` path: %s", err) @@ -105,8 +119,181 @@ func (c *Config) Validate() (err error) { } } + // Su is used to become the separated unprevileged user and control whom to become in sudoers + if c.SuPath == "" { + if c.SuPath, err = exec.LookPath("su"); err != nil { + return fmt.Errorf("Native: Unable to locate `su` path: %s", err) + } + } else { + if info, err := os.Stat(c.SuPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `su` path: %s, %s", c.SuPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `su` binary is not executable: %s", c.SuPath) + } + } + } + + // Sh is needed to set the unprevileged user default executable + if c.ShPath == "" { + if c.ShPath, err = exec.LookPath("sh"); err != nil { + return fmt.Errorf("Native: Unable to locate `su` path: %s", err) + } + } else { + if info, err := os.Stat(c.ShPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `sh` path: %s, %s", c.ShPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `sh` binary is not executable: %s", c.ShPath) + } + } + } + // Tar used to unpack the images + if c.TarPath == "" { + if c.TarPath, err = exec.LookPath("tar"); err != nil { + return fmt.Errorf("Native: Unable to locate `tar` path: %s", err) + } + } else { + if info, err := os.Stat(c.TarPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `tar` path: %s, %s", c.TarPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `tar` binary is not executable: %s", c.TarPath) + } + } + } + // Mount allows to look at the mounted volumes + if c.MountPath == "" { + if c.MountPath, err = exec.LookPath("mount"); err != nil { + return fmt.Errorf("Native: Unable to locate `mount` path: %s", err) + } + } else { + if info, err := os.Stat(c.MountPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `mount` path: %s, %s", c.MountPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `mount` binary is not executable: %s", c.MountPath) + } + } + } + // Chown needed to properly set ownership for the unprevileged user on available resources + if c.ChownPath == "" { + if c.ChownPath, err = exec.LookPath("chown"); err != nil { + return fmt.Errorf("Native: Unable to locate `chown` path: %s", err) + } + } else { + if info, err := os.Stat(c.ChownPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `chown` path: %s, %s", c.ChownPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `chown` binary is not executable: %s", c.ChownPath) + } + } + } + // Chmod needed to set additional read access for the unprevileged user on env metadata file + if c.ChmodPath == "" { + if c.ChmodPath, err = exec.LookPath("chmod"); err != nil { + return fmt.Errorf("Native: Unable to locate `chmod` path: %s", err) + } + } else { + if info, err := os.Stat(c.ChmodPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `chmod` path: %s, %s", c.ChmodPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `chmod` binary is not executable: %s", c.ChmodPath) + } + } + } + // Killall is running to stop all the unprevileged user processes during deallocation + if c.KillallPath == "" { + if c.KillallPath, err = exec.LookPath("killall"); err != nil { + return fmt.Errorf("Native: Unable to locate `killall` path: %s", err) + } + } else { + if info, err := os.Stat(c.KillallPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `killall` path: %s, %s", c.KillallPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `killall` binary is not executable: %s", c.KillallPath) + } + } + } + // Rm allows to clean up the leftowers after the execution + if c.RmPath == "" { + if c.RmPath, err = exec.LookPath("rm"); err != nil { + return fmt.Errorf("Native: Unable to locate `rm` path: %s", err) + } + } else { + if info, err := os.Stat(c.RmPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate `rm` path: %s, %s", c.RmPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: `rm` binary is not executable: %s", c.RmPath) + } + } + } + + // MacOS specific ones: + // Dscl creates/removes the unprevileged user + if c.DsclPath == "" { + if c.DsclPath, err = exec.LookPath("dscl"); err != nil { + return fmt.Errorf("Native: Unable to locate macos `dscl` path: %s", err) + } + } else { + if info, err := os.Stat(c.DsclPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate macos `dscl` path: %s, %s", c.DsclPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: macos `dscl` binary is not executable: %s", c.DsclPath) + } + } + } + // Hdiutil allows to create disk images and mount them to restrict user by disk space + if c.HdiutilPath == "" { + if c.HdiutilPath, err = exec.LookPath("hdiutil"); err != nil { + return fmt.Errorf("Native: Unable to locate macos `hdiutil` path: %s", err) + } + } else { + if info, err := os.Stat(c.HdiutilPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate macos `hdiutil` path: %s, %s", c.HdiutilPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: macos `hdiutil` binary is not executable: %s", c.HdiutilPath) + } + } + } + // Mdutil allows to disable the indexing for mounted volume + if c.MdutilPath == "" { + if c.MdutilPath, err = exec.LookPath("mdutil"); err != nil { + return fmt.Errorf("Native: Unable to locate macos `mdutil` path: %s", err) + } + } else { + if info, err := os.Stat(c.MdutilPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate macos `mdutil` path: %s, %s", c.MdutilPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: macos `mdutil` binary is not executable: %s", c.MdutilPath) + } + } + } + // Createhomedir creates unprevileged user home directory and fulfills with default subdirs + if c.CreatehomedirPath == "" { + if c.CreatehomedirPath, err = exec.LookPath("createhomedir"); err != nil { + return fmt.Errorf("Native: Unable to locate macos `createhomedir` path: %s", err) + } + } else { + if info, err := os.Stat(c.CreatehomedirPath); os.IsNotExist(err) { + return fmt.Errorf("Native: Unable to locate macos `createhomedir` path: %s, %s", c.CreatehomedirPath, err) + } else { + if info.Mode()&0111 == 0 { + return fmt.Errorf("Native: macos `createhomedir` binary is not executable: %s", c.CreatehomedirPath) + } + } + } + // Verify the configuration works for this machine var opts Options + opts.Validate() // If the users are not set - the user will be created dynamically // with "fish-" prefix and it's needed quite a good amount of access: @@ -138,7 +325,7 @@ func (c *Config) Validate() (err error) { // Clean after the run if err = userDelete(c, user); err != nil { - return fmt.Errorf("Native: Unable to delete user %q: %v", user, err) + return fmt.Errorf("Native: Unable to delete user in the end of driver verification %q: %v", user, err) } // TODO: diff --git a/lib/drivers/native/driver.go b/lib/drivers/native/driver.go index 4836377..2d10f95 100644 --- a/lib/drivers/native/driver.go +++ b/lib/drivers/native/driver.go @@ -198,7 +198,7 @@ func (d *Driver) Allocate(def types.LabelDefinition, metadata map[string]any) (* return nil, log.Error("Native: Unable to run the entry workload:", err) } - log.Info("Native: Started workload for user:", user, opts.Entry) + log.Infof("Native: Started environment for user %q", user) return &types.Resource{Identifier: user}, nil } diff --git a/lib/drivers/native/options.go b/lib/drivers/native/options.go index 8484ee8..07e7fa2 100644 --- a/lib/drivers/native/options.go +++ b/lib/drivers/native/options.go @@ -15,6 +15,7 @@ package native import ( "encoding/json" "fmt" + os_user "os/user" "runtime" "text/template" @@ -57,10 +58,10 @@ func (o *Options) Validate() error { // Set default entry if o.Entry == "" { if runtime.GOOS == "windows" { - o.Entry = "init.ps1" + o.Entry = ".\\init.ps1" } else { // On other systems sh should work just fine - o.Entry = "init.sh" + o.Entry = "./init.sh" } } // Verify that entry template is ok @@ -69,14 +70,17 @@ func (o *Options) Validate() error { } // Set default user groups + // The user is not complete without the primary group, so using current runtime user group if len(o.Groups) == 0 { - switch os := runtime.GOOS; os { - case "darwin": - o.Groups = append(o.Groups, "staff") - case "windows": - o.Groups = append(o.Groups, "Users") - // On the other systems user group will be created with the same name as user + u, e := os_user.Current() + if e != nil { + return log.Error("Native: Unable to get the current system user:", e) } + group, e := os_user.LookupGroupId(u.Gid) + if e != nil { + return log.Error("Native: Unable to get the current system user group name:", u.Gid, e) + } + o.Groups = append(o.Groups, group.Name) } // Check images diff --git a/lib/drivers/native/util.go b/lib/drivers/native/util.go index fc8f066..2c2fae1 100644 --- a/lib/drivers/native/util.go +++ b/lib/drivers/native/util.go @@ -20,6 +20,7 @@ import ( "io/fs" "os" "os/exec" + os_user "os/user" "path/filepath" "strconv" "strings" @@ -27,10 +28,13 @@ import ( "text/template" "time" + "github.com/alessio/shellescape" + "github.com/adobe/aquarium-fish/lib/crypt" "github.com/adobe/aquarium-fish/lib/drivers" "github.com/adobe/aquarium-fish/lib/log" "github.com/adobe/aquarium-fish/lib/openapi/types" + "github.com/adobe/aquarium-fish/lib/util" ) // Common lock to properly acquire unique User ID @@ -116,7 +120,7 @@ func (d *Driver) loadImages(user string, images []drivers.Image, disk_paths map[ } defer f.Close() log.Info("Native: Unpacking image:", user, image_archive, unpack_path) - _, _, err = runAndLog(5*time.Minute, f, d.cfg.SudoPath, "-n", "/usr/bin/tar", "-xf", "-", "--uname", user, "-C", unpack_path+"/") + _, _, err = runAndLog(5*time.Minute, f, d.cfg.SudoPath, "-n", d.cfg.TarPath, "-xf", "-", "--uname", user, "-C", unpack_path+"/") if err != nil { return log.Error("Native: Unable to unpack the image:", image_archive, err) } @@ -142,13 +146,13 @@ func userCreate(c *Config, groups []string) (user, homedir string, err error) { // In theory we can use `sysadminctl -addUser` command instead, but it asks for elevated previleges // so not sure how useful it will be in automation... - if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "create", "/Users/"+user, "RealName", "Aquarium Fish env user"); err != nil { + if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "create", "/Users/"+user, "RealName", "Aquarium Fish env user"); err != nil { err = log.Error("Native: Error user set RealName:", err) return } // Configure default shell - if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "create", "/Users/"+user, "UserShell", "/bin/sh"); err != nil { + if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "create", "/Users/"+user, "UserShell", c.ShPath); err != nil { err = log.Error("Native: Error user set UserShell:", err) return } @@ -158,7 +162,7 @@ func userCreate(c *Config, groups []string) (user, homedir string, err error) { { // Locate the unassigned user id var stdout string - if stdout, _, err = runAndLog(5*time.Second, nil, "/usr/bin/dscl", ".", "list", "/Users", "UniqueID"); err != nil { + if stdout, _, err = runAndLog(5*time.Second, nil, c.DsclPath, ".", "list", "/Users", "UniqueID"); err != nil { user_create_lock.Unlock() err = log.Error("Native: Unable to list directory users:", err) return @@ -180,7 +184,7 @@ func userCreate(c *Config, groups []string) (user, homedir string, err error) { } // Increment max user id and use it as unique id for new user - if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "create", "/Users/"+user, "UniqueID", fmt.Sprint(user_id+1)); err != nil { + if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "create", "/Users/"+user, "UniqueID", fmt.Sprint(user_id+1)); err != nil { user_create_lock.Unlock() err = log.Error("Native: Unable to set user UniqueID:", err) return @@ -188,59 +192,38 @@ func userCreate(c *Config, groups []string) (user, homedir string, err error) { } user_create_lock.Unlock() - if len(groups) > 0 { - // Locate the primary user group id - var stdout string - if stdout, _, err = runAndLog(5*time.Second, nil, "/usr/bin/dscl", ".", "list", "/Groups", "PrimaryGroupID"); err != nil { - err = log.Error("Native: Unable to list directory groups:", err) - return - } - - // Finding the primary group id in the list - primary_group_id := int64(-1) - split_stdout := strings.Split(strings.TrimSpace(stdout), "\n") - for _, line := range split_stdout { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, groups[0]+" ") { - line_id := line[strings.LastIndex(line, " ")+1:] - if primary_group_id, err = strconv.ParseInt(line_id, 10, 64); err != nil { - err = log.Error("Native: Unable to parse group id in line:", line) - return - } - break - } - } - - if primary_group_id == -1 { - err = log.Error("Native: Unable to find id for group:", groups[0]) - } + // Locate the primary user group id + primary_group, e := os_user.LookupGroup(groups[0]) + if e != nil { + err = log.Error("Native: Unable to locate group GID for:", groups[0], e) + return + } - // Set user primary group - if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "create", "/Users/"+user, "PrimaryGroupID", fmt.Sprint(primary_group_id)); err != nil { - err = log.Error("Native: Unable to set user PrimaryGroupID:", err) - return - } + // Set user primary group + if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "create", "/Users/"+user, "PrimaryGroupID", primary_group.Gid); err != nil { + err = log.Error("Native: Unable to set user PrimaryGroupID:", err) + return + } - // If there are other groups required - add user to them too - if len(groups) > 1 { - for _, group := range groups[1:] { - if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "append", "/Groups/"+group, "GroupMembership", user); err != nil { - err = log.Error("Native: Unable to add user to group:", group, err) - return - } + // If there are other groups required - add user to them too + if len(groups) > 1 { + for _, group := range groups[1:] { + if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "append", "/Groups/"+group, "GroupMembership", user); err != nil { + err = log.Error("Native: Unable to add user to group:", group, err) + return } } } // Set the default home directory homedir = filepath.Join("/Users", user) - if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "create", "/Users/"+user, "NFSHomeDirectory", homedir); err != nil { + if _, _, err = runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "create", "/Users/"+user, "NFSHomeDirectory", homedir); err != nil { err = log.Error("Native: Unable to set user NFSHomeDirectory:", err) return } // Populate the default user home directory - if _, _, err = runAndLog(30*time.Second, nil, c.SudoPath, "-n", "/usr/sbin/createhomedir", "-c", "-u", user); err != nil { + if _, _, err = runAndLog(30*time.Second, nil, c.SudoPath, "-n", c.CreatehomedirPath, "-c", "-u", user); err != nil { err = log.Error("Native: Unable to populate the default user directory:", err) return } @@ -275,24 +258,77 @@ func userRun(c *Config, env_data *EnvData, user, entry string, metadata map[stri } entry = tmp_data - // Prepare the command to execute entry from user home directory - cmd := exec.Command(c.SudoPath, "-n", "/usr/bin/su", "-l", user, entry) - // Metadata values could contain template data + env_vars := make(map[string]any) for key, val := range metadata { if tmp_data, err = processTemplate(env_data, fmt.Sprintf("%v", val)); err != nil { return log.Errorf("Native: Unable to process metadata `%s` template: %v", key, err) } - // Fill the command env vars - cmd.Env = append(cmd.Environ(), fmt.Sprintf("%s=%v", key, tmp_data)) + // Add to the map of the variables to store + env_vars[key] = tmp_data + } + + // Unfortunately passing the environment through the cmd.Env and sudo/su is not that easy, so + // using a temp file instead, which is removed right after the entry is started. + env_file_data, err := util.SerializeMetadata("export", "", env_vars) + if err != nil { + return log.Errorf("Native: Unable to serialize metadata into 'export' format: %v", err) + } + // Using common /tmp dir available for each user in the system + env_file, err := os.CreateTemp("/tmp", "*.metadata.sh") + if err != nil { + return log.Error("Native: Unable to create temp env file:", err) + } + defer os.Remove(env_file.Name()) + if _, err := env_file.Write(env_file_data); err != nil { + return log.Error("Native: Unable to write temp env file:", err) + } + if err := env_file.Close(); err != nil { + return log.Error("Native: Unable to close temp env file:", err) + } + + // Add ACL permission to the env file to allow to read it by unprevileged user + if _, _, err := runAndLogRetry(5, 5*time.Second, nil, c.ChmodPath, "+a", fmt.Sprintf("user:%s:allow read,readattr,readextattr,readsecurity", user), env_file.Name()); err != nil { + return log.Error("Native: Unable to set ACL for temp env file:", err) + } + + // Prepare the command to execute entry from user home directory + shell_line := fmt.Sprintf("source %s; %s", env_file.Name(), shellescape.Quote(shellescape.StripUnsafe(entry))) + cmd := exec.Command(c.SudoPath, "-n", c.SuPath, "-l", user, "-c", shell_line) + if env_data != nil && env_data.Disks != nil { + if _, ok := env_data.Disks[""]; ok { + cmd.Dir = env_data.Disks[""] + } + } + + // Printing stdout/stderr with proper prefix + cmd.Stdout = &util.StreamLogMonitor{ + Prefix: fmt.Sprintf("%s: ", user), + } + cmd.Stderr = &util.StreamLogMonitor{ + Prefix: fmt.Sprintf("%s: ", user), } // Run the process in background, it should live even when the Fish node is down if err = cmd.Start(); err != nil { return log.Error("Native: Unable to run the process:", err) } + // TODO: Probably I should run cmd.Wait to make sure the captured OS resources are released, + // but not sure about that... Maybe create a goroutine that will sit and wait there? - return nil + log.Debugf("Native: Started entry for user %q in directory %q with PID %d: %s", user, cmd.Dir, cmd.Process.Pid, shell_line) + + // Giving the process 1 second to read the env file and not die from some unexpected error + time.Sleep(time.Second) + if cmd.Err != nil { + err = log.Error("Native: The process ended quickly with error:", user, cmd.Err) + } + + if cmd.ProcessState != nil && !cmd.ProcessState.Success() { + err = log.Error("Native: The process ended quickly with non-zero exit code:", user, cmd.ProcessState.ExitCode(), cmd.ProcessState.Pid(), cmd.ProcessState.SystemTime(), cmd.ProcessState.UserTime(), cmd.ProcessState.String()) + } + + return err } // Stop the user processes @@ -304,15 +340,15 @@ func userStop(c *Config, user string) (out_err error) { // Note: some operations may fail, but they should not interrupt the whole cleanup process // Interrupt all the user processes - if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/killall", "-INT", "-u", user); err != nil { - log.Warn("Native: Unable to interrupt the user apps:", err) + if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.KillallPath, "-INT", "-u", user); err != nil { + log.Debug("Native: Unable to interrupt the user apps:", user, err) } // Check if no apps are running after interrupt - ps will end up with error if there is none apps left if _, _, err := runAndLog(5*time.Second, nil, "ps", "-U", user); err == nil { // Some apps are still running - give them 5 seconds to complete their processes time.Sleep(5 * time.Second) - if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/killall", "-KILL", "-u", user); err != nil { - log.Warn("Native: Unable to kill the user apps:", err) + if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.KillallPath, "-KILL", "-u", user); err != nil { + log.Warn("Native: Unable to kill the user apps:", user, err) } } @@ -324,11 +360,13 @@ func userDelete(c *Config, user string) (out_err error) { // Stopping the processes because they could cause user lock out_err = userStop(c, user) - if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/usr/bin/dscl", ".", "delete", "/Users/"+user); err != nil { + // Sometimes delete of the user could not be done due to MacOS blocking it, so retrying 5 times + // Native: Command exited with error: exit status 40:
delete status: eDSPermissionError DS Error: -14120 (eDSPermissionError) + if _, _, err := runAndLogRetry(5, 5*time.Second, nil, c.SudoPath, "-n", c.DsclPath, ".", "delete", "/Users/"+user); err != nil { out_err = log.Error("Native: Unable to delete user:", err) } - if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", "/bin/rm", "-rf", "/Users/"+user); err != nil { + if _, _, err := runAndLog(5*time.Second, nil, c.SudoPath, "-n", c.RmPath, "-rf", "/Users/"+user); err != nil { out_err = log.Error("Native: Unable to remove the user home directory:", err) } @@ -353,13 +391,13 @@ func disksDelete(c *Config, user string) (out_err error) { } // Umount the disk volumes if needed - mounts, _, err := runAndLog(3*time.Second, nil, "/sbin/mount") + mounts, _, err := runAndLog(3*time.Second, nil, c.MountPath) if err != nil { out_err = log.Error("Native: Unable to list the mount points:", user, err) } for _, vol_path := range env_volumes { if strings.Contains(mounts, vol_path) { - if _, _, err := runAndLog(5*time.Second, nil, "/usr/bin/hdiutil", "detach", vol_path); err != nil { + if _, _, err := runAndLog(5*time.Second, nil, c.HdiutilPath, "detach", vol_path); err != nil { out_err = log.Error("Native: Unable to detach the volume disk:", user, vol_path, err) } } @@ -418,7 +456,7 @@ func (d *Driver) disksCreate(user string, disks map[string]types.ResourcesDisk) "-volname", label, "-size", fmt.Sprintf("%dm", disk.Size*1024), } - if _, _, err := runAndLog(10*time.Minute, nil, "/usr/bin/hdiutil", args...); err != nil { + if _, _, err := runAndLog(10*time.Minute, nil, d.cfg.HdiutilPath, args...); err != nil { return disk_paths, log.Error("Native: Unable to create dmg disk:", dmg_path, err) } } @@ -426,17 +464,17 @@ func (d *Driver) disksCreate(user string, disks map[string]types.ResourcesDisk) mount_point := filepath.Join("/Volumes", fmt.Sprintf("%s_%s", user, d_name)) // Attach & mount disk - if _, _, err := runAndLog(10*time.Second, nil, "/usr/bin/hdiutil", "attach", dmg_path, "-owners", "on", "-mountpoint", mount_point); err != nil { + if _, _, err := runAndLog(10*time.Second, nil, d.cfg.HdiutilPath, "attach", dmg_path, "-owners", "on", "-mountpoint", mount_point); err != nil { return disk_paths, log.Error("Native: Unable to attach dmg disk:", dmg_path, mount_point, err) } // Change the owner of the volume to user - if _, _, err := runAndLog(5*time.Second, nil, d.cfg.SudoPath, "-n", "/usr/sbin/chown", "-R", user+":staff", mount_point+"/"); err != nil { + if _, _, err := runAndLog(5*time.Second, nil, d.cfg.SudoPath, "-n", d.cfg.ChownPath, "-R", user+":staff", mount_point+"/"); err != nil { return disk_paths, fmt.Errorf("Native: Error user disk mount path chown: %v", err) } // (Optional) Disable spotlight for the mounted volume - if _, _, err := runAndLog(5*time.Second, nil, d.cfg.SudoPath, "/usr/bin/mdutil", "-i", "off", mount_point+"/"); err != nil { + if _, _, err := runAndLog(5*time.Second, nil, d.cfg.SudoPath, d.cfg.MdutilPath, "-i", "off", mount_point+"/"); err != nil { log.Warn("Native: Unable to disable spotlight for the volume:", mount_point, err) } @@ -492,3 +530,24 @@ func runAndLog(timeout time.Duration, stdin io.Reader, path string, arg ...strin return returnStdout, returnStderr, err } + +// Will retry on error and store the retry output and errors to return +func runAndLogRetry(retry int, timeout time.Duration, stdin io.Reader, path string, arg ...string) (stdout string, stderr string, err error) { + counter := 0 + for { + counter++ + rout, rerr, err := runAndLog(timeout, stdin, path, arg...) + if err != nil { + stdout += fmt.Sprintf("\n--- Fish: Command execution attempt %d ---\n", counter) + stdout += rout + stderr += fmt.Sprintf("\n--- Fish: Command execution attempt %d ---\n", counter) + stderr += rerr + if counter <= retry { + // Give command time to rest + time.Sleep(time.Duration(counter) * time.Second) + continue + } + } + return stdout, stderr, err + } +} diff --git a/lib/util/metadata_processing.go b/lib/util/metadata_processing.go index da8972c..a49f22b 100644 --- a/lib/util/metadata_processing.go +++ b/lib/util/metadata_processing.go @@ -23,6 +23,17 @@ func SerializeMetadata(format, prefix string, data map[string]any) (out []byte, value := []byte("=" + shellescape.Quote(val) + "\n") out = append(out, append(line, value...)...) } + case "export": // Format env with exports for easy usage with source + m := DotSerialize(prefix, data) + for key, val := range m { + line := cleanShellKey(strings.Replace(shellescape.StripUnsafe(key), ".", "_", -1)) + if len(line) == 0 { + continue + } + line = append([]byte("export "), line...) + value := []byte("=" + shellescape.Quote(val) + "\n") + out = append(out, append(line, value...)...) + } case "ps1": // Plain format suitable to use in powershell m := DotSerialize(prefix, data) for key, val := range m { diff --git a/lib/util/streamlog_monitor.go b/lib/util/streamlog_monitor.go new file mode 100644 index 0000000..aee40b2 --- /dev/null +++ b/lib/util/streamlog_monitor.go @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +package util + +import ( + "bytes" + + "github.com/adobe/aquarium-fish/lib/log" +) + +var LineBreak = []byte("\n") +var EmptyByte = []byte{} + +// Wraps an existing io.Reader to monitor the log stream and adds prefix before each line +type StreamLogMonitor struct { + Prefix string // Prefix for the line + linebuf [][]byte // Where line will live until EOL or close +} + +// Read 'overrides' the underlying io.Reader's Read method +func (slm *StreamLogMonitor) Write(p []byte) (int, error) { + index := 0 + prev_index := 0 + for index < len(p) { + index += bytes.Index(p[prev_index:], LineBreak) + if index == -1 { + // The data does not contain EOL, so appending to buffer and wait + slm.linebuf = append(slm.linebuf, p) + break + } + // The newline was found, so prepending the line buffer and print it out + // We don't need the EOF in the line (log.Infof adds), so increment index after processing + slm.linebuf = append(slm.linebuf, p[prev_index:index]) + log.Info(slm.Prefix + string(bytes.Join(slm.linebuf, EmptyByte))) + clear(slm.linebuf) + index++ + prev_index = index + } + + return len(p), nil +} diff --git a/tests/allocate_multidefinition_label_test.go b/tests/allocate_multidefinition_label_test.go index d1b98ea..a89916d 100644 --- a/tests/allocate_multidefinition_label_test.go +++ b/tests/allocate_multidefinition_label_test.go @@ -23,13 +23,13 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Checks the labels processing by having 3 different definitions where one is suitable enough func Test_allocate_multidefinition_label(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -99,7 +99,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -145,7 +145,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 5 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -199,7 +199,7 @@ drivers: }) t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -244,7 +244,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 5 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -299,7 +299,7 @@ drivers: }) t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -344,7 +344,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 5 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/application_task_notexisting_fail_test.go b/tests/application_task_notexisting_fail_test.go index 7c26dec..434b177 100644 --- a/tests/application_task_notexisting_fail_test.go +++ b/tests/application_task_notexisting_fail_test.go @@ -23,13 +23,13 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Ensure application task could be created with weird name but will fail during execution func Test_application_task_notexisting_fail(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -91,7 +91,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -126,7 +126,7 @@ drivers: var app_tasks []types.ApplicationTask t.Run("ApplicationTask should be executed as not found in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/task/")). @@ -159,7 +159,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/application_task_snapshot_by_user_test.go b/tests/application_task_snapshot_by_user_test.go index 2cc6845..9bd0636 100644 --- a/tests/application_task_snapshot_by_user_test.go +++ b/tests/application_task_snapshot_by_user_test.go @@ -23,13 +23,13 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Ensure application task interface snapshot for user is working func Test_application_task_snapshot_by_user(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -107,7 +107,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -175,7 +175,7 @@ drivers: var app_tasks []types.ApplicationTask t.Run("ApplicationTask 1 should be executed in 10 sec and 2 should not be executed", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/task/")). @@ -214,7 +214,7 @@ drivers: }) t.Run("ApplicationTask 2 should be executed in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/task/")). @@ -237,7 +237,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/cant_allocate_too_big_label_test.go b/tests/cant_allocate_too_big_label_test.go index 36ae34d..8f50536 100644 --- a/tests/cant_allocate_too_big_label_test.go +++ b/tests/cant_allocate_too_big_label_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Checks that the node will not try to execute the bigger Label Application: @@ -32,8 +33,7 @@ import ( // * Should get RECALLED state func Test_cant_allocate_too_big_label(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -176,7 +176,7 @@ drivers: }) t.Run("Application should get RECALLED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/default_lifetime_timeout_test.go b/tests/default_lifetime_timeout_test.go index 93b424a..42aa60a 100644 --- a/tests/default_lifetime_timeout_test.go +++ b/tests/default_lifetime_timeout_test.go @@ -23,13 +23,13 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Checks the Application is getting deallocated by default timeout in node config func Test_default_lifetime_timeout(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc default_resource_lifetime: 15s @@ -94,7 +94,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -128,7 +128,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 5 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/generated_uids_prefix_is_node_prefix_test.go b/tests/generated_uids_prefix_is_node_prefix_test.go index 9614d50..9aa3c3e 100644 --- a/tests/generated_uids_prefix_is_node_prefix_test.go +++ b/tests/generated_uids_prefix_is_node_prefix_test.go @@ -24,6 +24,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // The test ensures the Node is actually participate in generating it's data UID's @@ -34,8 +35,7 @@ import ( // * TODO: Other data UIDs func Test_generated_uids_prefix_is_node_prefix(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -126,7 +126,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -185,7 +185,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/helper/copy.go b/tests/helper/copy.go new file mode 100644 index 0000000..aacfad8 --- /dev/null +++ b/tests/helper/copy.go @@ -0,0 +1,41 @@ +/** + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +package helper + +import ( + "io" + "os" + "path/filepath" +) + +// Copy files around +func CopyFile(src, dst string) error { + fin, err := os.Open(src) + if err != nil { + return err + } + defer fin.Close() + + os.MkdirAll(filepath.Dir(dst), 0755) + fout, err := os.Create(dst) + if err != nil { + return err + } + defer fout.Close() + + if _, err = io.Copy(fout, fin); err != nil { + return err + } + + return nil +} diff --git a/tests/helper/fish.go b/tests/helper/fish.go new file mode 100644 index 0000000..46c64f7 --- /dev/null +++ b/tests/helper/fish.go @@ -0,0 +1,220 @@ +/** + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +package helper + +import ( + "bufio" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +var fish_path = os.Getenv("FISH_PATH") // Full path to the aquarium-fish binary + +// Saves state of the running Aquarium Fish for particular test +type afInstance struct { + workspace string + fishKill context.CancelFunc + running bool + cmd *exec.Cmd + + node_name string + endpoint string + admin_token string +} + +// Simple creates and run the fish node +func NewAquariumFish(t testing.TB, name, cfg string, args ...string) *afInstance { + afi := NewAfInstance(t, name, cfg) + afi.Start(t, args...) + + return afi +} + +// If you need to create instance without starting it up right away +func NewAfInstance(t testing.TB, name, cfg string) *afInstance { + t.Log("INFO: Creating new node:", name) + afi := &afInstance{ + node_name: name, + } + + afi.workspace = t.TempDir() + t.Log("INFO: Created workspace:", afi.node_name, afi.workspace) + + cfg += fmt.Sprintf("\nnode_name: %q", afi.node_name) + os.WriteFile(filepath.Join(afi.workspace, "config.yml"), []byte(cfg), 0644) + t.Log("INFO: Stored config:", cfg) + + return afi +} + +// Start another node of cluster +// It will automatically add cluster_join parameter to the config +func (afi1 *afInstance) NewClusterNode(t testing.TB, name, cfg string, args ...string) *afInstance { + afi2 := afi1.NewAfInstanceCluster(t, name, cfg) + afi2.Start(t, args...) + + return afi2 +} + +// Just create the node based on the existing cluster node +func (afi1 *afInstance) NewAfInstanceCluster(t testing.TB, name, cfg string) *afInstance { + t.Log("INFO: Creating new cluster node with seed node:", afi1.node_name) + cfg += fmt.Sprintf("\ncluster_join: [%q]", afi1.endpoint) + afi2 := NewAfInstance(t, name, cfg) + + // Copy seed node CA to generate valid cluster node cert + if err := CopyFile(filepath.Join(afi1.workspace, "fish_data", "ca.key"), filepath.Join(afi2.workspace, "fish_data", "ca.key")); err != nil { + t.Fatalf("ERROR: Unable to copy CA key: %v", err) + } + if err := CopyFile(filepath.Join(afi1.workspace, "fish_data", "ca.crt"), filepath.Join(afi2.workspace, "fish_data", "ca.crt")); err != nil { + t.Fatalf("ERROR: Unable to copy CA crt: %v", err) + } + + return afi2 +} + +// Will return just IP:PORT +func (afi *afInstance) Endpoint() string { + return afi.endpoint +} + +// Will return url to access API of AquariumFish +func (afi *afInstance) ApiAddress(path string) string { + return fmt.Sprintf("https://%s/%s", afi.endpoint, path) +} + +// Will return workspace of the AquariumFish +func (afi *afInstance) Workspace() string { + return afi.workspace +} + +// Returns admin token +func (afi *afInstance) AdminToken() string { + return afi.admin_token +} + +// Check the fish instance is running +func (afi *afInstance) IsRunning() bool { + return afi.running +} + +// Restart the application +func (afi *afInstance) Restart(t testing.TB, args ...string) { + t.Log("INFO: Restarting:", afi.node_name, afi.workspace) + afi.Stop(t) + afi.Start(t, args...) +} + +// Cleanup after the test execution +func (afi *afInstance) Cleanup(t testing.TB) { + t.Log("INFO: Cleaning up:", afi.node_name, afi.workspace) + afi.Stop(t) + os.RemoveAll(afi.workspace) +} + +// Stops the fish node executable +func (afi *afInstance) Stop(t testing.TB) { + if afi.cmd == nil || !afi.running { + return + } + // Send interrupt signal + afi.cmd.Process.Signal(os.Interrupt) + + // Wait 10 seconds for process to stop + t.Log("INFO: Wait 10s for fish node to stop:", afi.node_name, afi.workspace) + for i := 1; i < 20; i++ { + if !afi.running { + return + } + time.Sleep(50 * time.Millisecond) + } + + // Hard killing the process + afi.fishKill() +} + +// Starts the fish node executable +func (afi *afInstance) Start(t testing.TB, args ...string) { + if afi.running { + t.Fatalf("ERROR: Fish node %q can't be started since already started", afi.node_name) + return + } + ctx, cancel := context.WithCancel(context.Background()) + afi.fishKill = cancel + + cmd_args := []string{"-v", "debug", "-c", filepath.Join(afi.workspace, "config.yml")} + cmd_args = append(cmd_args, args...) + afi.cmd = exec.CommandContext(ctx, fish_path, cmd_args...) + afi.cmd.Dir = afi.workspace + r, _ := afi.cmd.StdoutPipe() + afi.cmd.Stderr = afi.cmd.Stdout + + init_done := make(chan string) + scanner := bufio.NewScanner(r) + // TODO: Add timeout for waiting of API available + go func() { + // Listening for log and scan for token and address + for scanner.Scan() { + line := scanner.Text() + t.Log(afi.node_name, line) + if strings.HasPrefix(line, "Admin user pass: ") { + val := strings.SplitN(strings.TrimSpace(line), "Admin user pass: ", 2) + if len(val) < 2 { + init_done <- "ERROR: No token after 'Admin user pass: '" + break + } + afi.admin_token = val[1] + } + if strings.Contains(line, "API listening on: ") { + val := strings.SplitN(strings.TrimSpace(line), "API listening on: ", 2) + if len(val) < 2 { + init_done <- "ERROR: No address after 'API listening on: '" + break + } + afi.endpoint = val[1] + } + if strings.HasSuffix(line, "Fish initialized") { + // Found the needed values and continue to process to print the fish output for + // test debugging purposes + init_done <- "" + } + } + t.Log("INFO: Reading of AquariumFish output is done") + }() + + afi.cmd.Start() + + go func() { + afi.running = true + defer func() { + afi.running = false + r.Close() + }() + if err := afi.cmd.Wait(); err != nil { + t.Log("WARN: AquariumFish process was stopped:", err) + init_done <- fmt.Sprintf("ERROR: Fish was stopped with exit code: %v", err) + } + }() + + failed := <-init_done + + if failed != "" { + t.Fatalf("ERROR: Failed to init node %q: %s", afi.node_name, failed) + } +} diff --git a/tests/helper/retry.go b/tests/helper/retry.go new file mode 100644 index 0000000..bcaa5ee --- /dev/null +++ b/tests/helper/retry.go @@ -0,0 +1,207 @@ +/** + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +package helper + +import ( + "bytes" + "fmt" + "runtime" + "strings" + "time" +) + +// Failer is an interface compatible with testing.T. +type Failer interface { + Helper() + + // Log is called for the final test output + Log(args ...any) + + // FailNow is called when the retrying is abandoned. + FailNow() +} + +// R provides context for the retryer. +type R struct { + fail bool + done bool + output []string +} + +func (r *R) Helper() {} + +var runFailed = struct{}{} + +func (r *R) FailNow() { + r.fail = true + panic(runFailed) +} + +func (r *R) Fatal(args ...any) { + r.log(fmt.Sprint(args...)) + r.FailNow() +} + +func (r *R) Fatalf(format string, args ...any) { + r.log(fmt.Sprintf(format, args...)) + r.FailNow() +} + +func (r *R) Error(args ...any) { + r.log(fmt.Sprint(args...)) + r.fail = true +} + +func (r *R) Errorf(format string, args ...any) { + r.log(fmt.Sprintf(format, args...)) + r.fail = true +} + +func (r *R) Check(err error) { + if err != nil { + r.log(err.Error()) + r.FailNow() + } +} + +func (r *R) log(s string) { + r.output = append(r.output, decorate(s)) +} + +// Stop retrying, and fail the test with the specified error. +func (r *R) Stop(err error) { + r.log(err.Error()) + r.done = true +} + +func decorate(s string) string { + _, file, line, ok := runtime.Caller(3) + if ok { + n := strings.LastIndex(file, "/") + if n >= 0 { + file = file[n+1:] + } + } else { + file = "???" + line = 1 + } + return fmt.Sprintf("%s:%d: %s", file, line, s) +} + +func Retry(r Retryer, t Failer, f func(r *R)) { + t.Helper() + run(r, t, f) +} + +func dedup(a []string) string { + if len(a) == 0 { + return "" + } + seen := map[string]struct{}{} + var b bytes.Buffer + for _, s := range a { + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + b.WriteString(s) + b.WriteRune('\n') + } + return b.String() +} + +func run(r Retryer, t Failer, f func(r *R)) { + t.Helper() + rr := &R{} + + fail := func() { + t.Helper() + out := dedup(rr.output) + if out != "" { + t.Log(out) + } + t.FailNow() + } + + for r.Continue() { + func() { + defer func() { + if p := recover(); p != nil && p != runFailed { + panic(p) + } + }() + f(rr) + }() + + switch { + case rr.done: + fail() + return + case !rr.fail: + return + } + rr.fail = false + } + fail() +} + +// Retryer provides an interface for repeating operations +// until they succeed or an exit condition is met. +type Retryer interface { + // Continue returns true if the operation should be repeated, otherwise it + // returns false to indicate retrying should stop. + Continue() bool +} + +// Counter repeats an operation a given number of +// times and waits between subsequent operations. +type Counter struct { + Count int + Wait time.Duration + + count int +} + +func (r *Counter) Continue() bool { + if r.count == r.Count { + return false + } + if r.count > 0 { + time.Sleep(r.Wait) + } + r.count++ + return true +} + +// Timer repeats an operation for a given amount +// of time and waits between subsequent operations. +type Timer struct { + Timeout time.Duration + Wait time.Duration + + // stop is the timeout deadline. + // Set on the first invocation of Next(). + stop time.Time +} + +func (r *Timer) Continue() bool { + if r.stop.IsZero() { + r.stop = time.Now().Add(r.Timeout) + return true + } + if time.Now().After(r.stop) { + return false + } + time.Sleep(r.Wait) + return true +} diff --git a/tests/helper/t_mock.go b/tests/helper/t_mock.go new file mode 100644 index 0000000..de19e45 --- /dev/null +++ b/tests/helper/t_mock.go @@ -0,0 +1,67 @@ +/** + * Copyright 2023 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +package helper + +import ( + "runtime" + "sync" + "testing" +) + +// Useful to capture the failed test +type MockT struct { + testing.T + + FailNowCalled bool + + t *testing.T +} + +func (m *MockT) FailNow() { + m.FailNowCalled = true + runtime.Goexit() +} + +func (m *MockT) Log(args ...any) { + m.t.Log(args...) +} + +func (m *MockT) Logf(format string, args ...any) { + m.t.Logf(format, args...) +} + +func (m *MockT) Fatal(args ...any) { + m.t.Log(args...) + m.FailNow() +} + +func (m *MockT) Fatalf(format string, args ...any) { + m.t.Logf(format, args...) + m.FailNow() +} + +func ExpectFailure(t *testing.T, f func(tt testing.TB)) { + var wg sync.WaitGroup + mock_t := &MockT{t: t} + + wg.Add(1) + go func() { + defer wg.Done() + f(mock_t) + }() + wg.Wait() + + if !mock_t.FailNowCalled { + t.Fatalf("ExpectFailure: the function did not fail as expected") + } +} diff --git a/tests/json_label_create_test.go b/tests/json_label_create_test.go index 4795d38..b6a0a83 100644 --- a/tests/json_label_create_test.go +++ b/tests/json_label_create_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // This is a test which makes sure we can send json input to create a Label @@ -30,8 +31,7 @@ import ( // * Check Label was created func Test_json_label_create(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 diff --git a/tests/label_find_filter_sql_injection_test.go b/tests/label_find_filter_sql_injection_test.go index 86e94a5..0232a01 100644 --- a/tests/label_find_filter_sql_injection_test.go +++ b/tests/label_find_filter_sql_injection_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Filters are using the user-defined SQL in find, so need to make sure there is no SQL injection @@ -31,8 +32,7 @@ import ( // * A number of valid filter requests func Test_label_find_filter_sql_injection(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 diff --git a/tests/label_lifetime_timeout_test.go b/tests/label_lifetime_timeout_test.go index 0e9bb7c..ac28315 100644 --- a/tests/label_lifetime_timeout_test.go +++ b/tests/label_lifetime_timeout_test.go @@ -23,13 +23,13 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Checks the Application is getting deallocated by timeout func Test_label_lifetime_timeout(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -93,7 +93,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -127,7 +127,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 5 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/label_overrides_default_lifetime_timeout_test.go b/tests/label_overrides_default_lifetime_timeout_test.go index 91c8024..a5cd8b0 100644 --- a/tests/label_overrides_default_lifetime_timeout_test.go +++ b/tests/label_overrides_default_lifetime_timeout_test.go @@ -23,13 +23,13 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Checks the Application is getting deallocated by label rather than default timeout in node config func Test_label_overrides_default_lifetime_timeout(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc default_resource_lifetime: 5s @@ -94,7 +94,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -128,7 +128,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 5 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 5 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/maintenance_mode_test.go b/tests/maintenance_mode_test.go index e4acf0f..f6c2fe5 100644 --- a/tests/maintenance_mode_test.go +++ b/tests/maintenance_mode_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Testing the maintenance cancel @@ -33,8 +34,7 @@ import ( // * Application should be allocated in 20 sec func Test_maintenace_cancel(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -134,7 +134,7 @@ drivers: }) t.Run("Application should get ALLOCATED in 20 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 20 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 20 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/node_filter_test.go b/tests/node_filter_test.go index cf4a2c2..103a90b 100644 --- a/tests/node_filter_test.go +++ b/tests/node_filter_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Make sure one node filter will match to allocate @@ -31,8 +32,7 @@ import ( // * Make sure it's allocated func Test_node_filter_one_match_good(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc node_identifiers: - "example:test" @@ -97,7 +97,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -120,8 +120,7 @@ drivers: // * Make sure it's allocated func Test_node_filter_two_match_good(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc node_identifiers: - "example:test" @@ -187,7 +186,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -210,8 +209,7 @@ drivers: // * Make sure it's not allocated in 10 sec func Test_node_filter_label_wrong_filter(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc node_identifiers: - "example:test" @@ -299,8 +297,7 @@ drivers: // * Make sure it's not allocated in 10 sec func Test_node_filter_node_has_less_than_needed(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc node_identifiers: - "example:test" diff --git a/tests/shutdown_mode_test.go b/tests/shutdown_mode_test.go index a2dd935..8e68b9a 100644 --- a/tests/shutdown_mode_test.go +++ b/tests/shutdown_mode_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Just regular maintenance + shutdown test: @@ -33,8 +34,7 @@ import ( // * Fish should shutdown in 10 sec func Test_shutdown_after_maintenace(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -96,7 +96,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -141,7 +141,7 @@ drivers: }) t.Run("Fish should stop in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { if afi.IsRunning() { r.Fatalf("Fish is still running, but should be stopped already") } @@ -155,8 +155,7 @@ drivers: // * Fish should shutdown in 10 sec func Test_immediate_shutdown(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -218,7 +217,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -247,7 +246,7 @@ drivers: }) t.Run("Fish should stop in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { if afi.IsRunning() { r.Fatalf("Fish is still running, but should be stopped already") } @@ -262,8 +261,7 @@ drivers: // * Fish should shutdown in 10 sec func Test_shutdown_after_delay(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -325,7 +323,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -362,7 +360,7 @@ drivers: }) t.Run("Fish should stop in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { if afi.IsRunning() { r.Fatalf("Fish is still running, but should be stopped already") } @@ -378,8 +376,7 @@ drivers: // * Fish should not shutdown in 10 sec func Test_shutdown_cancel_during_maintenance(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -441,7 +438,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -498,7 +495,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -530,8 +527,7 @@ drivers: // * Fish should not shutdown in 10 sec func Test_shutdown_cancel_during_delay(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 diff --git a/tests/simple_app_create_destroy_test.go b/tests/simple_app_create_destroy_test.go index 1cec308..3af26b2 100644 --- a/tests/simple_app_create_destroy_test.go +++ b/tests/simple_app_create_destroy_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // This is a simple test with one application and without any limits: @@ -31,8 +32,7 @@ import ( // * Destroy Application func Test_simple_app_create_destroy(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -94,7 +94,7 @@ drivers: var app_state types.ApplicationState t.Run("Application should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). @@ -137,7 +137,7 @@ drivers: }) t.Run("Application should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app.UID.String()+"/state")). diff --git a/tests/three_apps_with_limit_fish_restart_test.go b/tests/three_apps_with_limit_fish_restart_test.go index 75a76c8..0c45282 100644 --- a/tests/three_apps_with_limit_fish_restart_test.go +++ b/tests/three_apps_with_limit_fish_restart_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Will allocate 2 Applications and restart the fish node to check if they will be picked up after @@ -33,8 +34,7 @@ import ( // * Destroy the third app func Test_three_apps_with_limit_fish_restart(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -133,7 +133,7 @@ drivers: var app_state types.ApplicationState t.Run("Application 1 should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app1.UID.String()+"/state")). @@ -150,7 +150,7 @@ drivers: }) t.Run("Application 2 should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app2.UID.String()+"/state")). @@ -252,7 +252,7 @@ drivers: }) t.Run("Application 1 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app1.UID.String()+"/state")). @@ -269,7 +269,7 @@ drivers: }) t.Run("Application 2 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app2.UID.String()+"/state")). @@ -286,7 +286,7 @@ drivers: }) t.Run("Application 3 should get ALLOCATED in 40 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 40 * time.Second, Wait: 5 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 40 * time.Second, Wait: 5 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app3.UID.String()+"/state")). @@ -313,7 +313,7 @@ drivers: }) t.Run("Application 3 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app3.UID.String()+"/state")). diff --git a/tests/three_apps_with_limit_test.go b/tests/three_apps_with_limit_test.go index b34b8b1..69820eb 100644 --- a/tests/three_apps_with_limit_test.go +++ b/tests/three_apps_with_limit_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Will check how the Apps are allocated with limited amount of resources it should looks like: @@ -31,8 +32,7 @@ import ( // * Destroy the third app func Test_three_apps_with_limit(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -131,7 +131,7 @@ drivers: var app_state types.ApplicationState t.Run("Application 1 should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app1.UID.String()+"/state")). @@ -148,7 +148,7 @@ drivers: }) t.Run("Application 2 should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app2.UID.String()+"/state")). @@ -200,7 +200,7 @@ drivers: }) t.Run("Application 1 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app1.UID.String()+"/state")). @@ -217,7 +217,7 @@ drivers: }) t.Run("Application 2 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app2.UID.String()+"/state")). @@ -234,7 +234,7 @@ drivers: }) t.Run("Application 3 should get ALLOCATED in 40 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 40 * time.Second, Wait: 5 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 40 * time.Second, Wait: 5 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app3.UID.String()+"/state")). @@ -261,7 +261,7 @@ drivers: }) t.Run("Application 3 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app3.UID.String()+"/state")). diff --git a/tests/two_apps_with_limit_test.go b/tests/two_apps_with_limit_test.go index 4c2a521..f7b8ae4 100644 --- a/tests/two_apps_with_limit_test.go +++ b/tests/two_apps_with_limit_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // Checks the complete fill of the node with one Application, so the next one can't be executed: @@ -33,8 +34,7 @@ import ( // * Destroy second Application func Test_two_apps_with_limit(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0 @@ -116,7 +116,7 @@ drivers: var app_state types.ApplicationState t.Run("Application 1 should get ALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app1.UID.String()+"/state")). @@ -174,7 +174,7 @@ drivers: }) t.Run("Application 1 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app1.UID.String()+"/state")). @@ -191,7 +191,7 @@ drivers: }) t.Run("Application 2 should get ALLOCATED in 40 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 40 * time.Second, Wait: 5 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 40 * time.Second, Wait: 5 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app2.UID.String()+"/state")). @@ -233,7 +233,7 @@ drivers: }) t.Run("Application 2 should get DEALLOCATED in 10 sec", func(t *testing.T) { - Retry(&Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *R) { + h.Retry(&h.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}, t, func(r *h.R) { apitest.New(). EnableNetworking(cli). Get(afi.ApiAddress("api/v1/application/"+app2.UID.String()+"/state")). diff --git a/tests/yaml_label_create_test.go b/tests/yaml_label_create_test.go index c2b4c54..0f92519 100644 --- a/tests/yaml_label_create_test.go +++ b/tests/yaml_label_create_test.go @@ -23,6 +23,7 @@ import ( "github.com/steinfletcher/apitest" "github.com/adobe/aquarium-fish/lib/openapi/types" + h "github.com/adobe/aquarium-fish/tests/helper" ) // This is a test which makes sure we can send yaml input to create a Label @@ -30,8 +31,7 @@ import ( // * Check Label was created func Test_yaml_label_create(t *testing.T) { t.Parallel() - afi := RunAquariumFish(t, `--- -node_name: node-1 + afi := h.NewAquariumFish(t, "node-1", `--- node_location: test_loc api_address: 127.0.0.1:0