20160520

Development.md

@@ -184,6 +184,7 @@ Some general notes when making changes to the infrastructure templates:
 * Run `make test` to exercise the app template regression tests. Changes to app.tmpl almost always need accompanying test changes.
 * Pay careful attention to both the update and rollback safety of changes. Rollbacks are extremely important for failure recovery.
 * Convox uses [CloudFormation Custom Resources](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html) by releasing `api/cmd/formation` as a Lambda handler that every Rack and App can use to provision things that aren't supported by CloudFormation.
+* Run `make -C api/models/fixtures` to rebuild the test fixtures after you change manifests. Carefully inspect the diff to ensure only your desired changes are made.
 
 ## Opening a Pull Request
 

api/cmd/fixture/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/convox/rack/api/models"
+)
+
+func init() {
+	models.ManifestRandomPorts = false
+
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		die(fmt.Errorf("usage: fixture <docker-compose.yml>"))
+	}
+
+	data, err := ioutil.ReadFile(os.Args[1])
+
+	if err != nil {
+		die(err)
+	}
+
+	app := &models.App{
+		Name: "httpd",
+		Tags: map[string]string{
+			"Name":   "httpd",
+			"Type":   "app",
+			"System": "convox",
+			"Rack":   "convox",
+		},
+	}
+
+	m, err := models.LoadManifest(string(data), app)
+
+	if err != nil {
+		die(err)
+	}
+
+	f, err := m.Formation()
+
+	if err != nil {
+		die(err)
+	}
+
+	fmt.Println(f)
+}
+
+func die(err error) {
+	fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
+	os.Exit(1)
+}

api/manifest/manifest.go

@@ -463,8 +463,8 @@ func (m *Manifest) MissingEnvironment(cache bool, app string) ([]string, error)
 	return missing, nil
 }
 
-func (m *Manifest) PortConflicts() ([]string, error) {
-	wanted := m.PortsWanted()
+func (m *Manifest) PortConflicts(shift int) ([]string, error) {
+	wanted := m.PortsWanted(shift)
 
 	conflicts := make([]string, 0)
 
@@ -482,7 +482,7 @@ func (m *Manifest) PortConflicts() ([]string, error) {
 	return conflicts, nil
 }
 
-func (m *Manifest) PortsWanted() []string {
+func (m *Manifest) PortsWanted(shift int) []string {
 	ports := make([]string, 0)
 
 	for _, entry := range *m {
@@ -492,7 +492,8 @@ func (m *Manifest) PortsWanted() []string {
 					parts := strings.SplitN(p, ":", 2)
 
 					if len(parts) == 2 {
-						ports = append(ports, parts[0])
+						pi, _ := strconv.Atoi(parts[0])
+						ports = append(ports, fmt.Sprintf("%d", pi+shift))
 					}
 				}
 			}
@@ -545,7 +546,7 @@ func (m *Manifest) Raw() ([]byte, error) {
 	return yaml.Marshal(m)
 }
 
-func (m *Manifest) Run(app string, cache bool) []error {
+func (m *Manifest) Run(app string, cache bool, shift int) []error {
 	ch := make(chan error)
 	sigch := make(chan os.Signal, 1)
 	signal.Notify(sigch, os.Interrupt, os.Kill)
@@ -564,7 +565,7 @@ func (m *Manifest) Run(app string, cache bool) []error {
 
 	for i, name := range m.runOrder() {
 		sch := make(chan error)
-		go (*m)[name].runAsync(m, m.prefixForEntry(name, i), app, name, cache, ch, sch)
+		go (*m)[name].runAsync(m, m.prefixForEntry(name, i), app, name, cache, shift, ch, sch)
 		<-sch // block until started successfully
 	}
 
@@ -698,7 +699,7 @@ func containerName(app, process string) string {
 	return fmt.Sprintf("%s-%s", app, process)
 }
 
-func (me ManifestEntry) runAsync(m *Manifest, prefix, app, process string, cache bool, ch chan error, sch chan error) {
+func (me ManifestEntry) runAsync(m *Manifest, prefix, app, process string, cache bool, shift int, ch chan error, sch chan error) {
 	tag := fmt.Sprintf("%s/%s", app, process)
 	name := containerName(app, process)
 
@@ -734,11 +735,15 @@ func (me ManifestEntry) runAsync(m *Manifest, prefix, app, process string, cache
 		}
 	}
 
-	gateway, err := getDockerGateway()
+	if s := me.Label("convox.start.shift"); s != "" {
+		si, err := strconv.Atoi(s)
 
-	if err != nil {
-		ch <- err
-		return
+		if err != nil {
+			ch <- err
+			return
+		}
+
+		shift = si
 	}
 
 	host := ""
@@ -758,17 +763,31 @@ func (me ManifestEntry) runAsync(m *Manifest, prefix, app, process string, cache
 			return
 		}
 
+		hosti, err := strconv.Atoi(host)
+
+		if err != nil {
+			ch <- err
+			return
+		}
+
+		host = fmt.Sprintf("%d", hosti+shift)
+
 		switch proto := me.Label(fmt.Sprintf("convox.port.%s.protocol", host)); proto {
 		case "https", "tls":
 			proxy := false
+			secure := false
 
 			if me.Label(fmt.Sprintf("convox.port.%s.proxy", host)) == "true" {
 				proxy = true
 			}
 
+			if me.Label(fmt.Sprintf("convox.port.%s.secure", host)) == "true" {
+				secure = true
+			}
+
 			rnd := RandomPort()
 			fmt.Println(prefix, special(fmt.Sprintf("%s proxy enabled for %s:%s", proto, host, container)))
-			go proxyPort(proto, host, fmt.Sprintf("%s:%d", gateway, rnd), proxy)
+			go proxyPort(proto, host, container, name, proxy, secure)
 			host = strconv.Itoa(rnd)
 		}
 
@@ -888,7 +907,7 @@ func (me ManifestEntry) syncAdds(app, process string) error {
 	}
 
 	for _, line := range strings.Split(string(data), "\n") {
-		parts := strings.Split(strings.TrimSpace(line), " ")
+		parts := strings.Fields(line)
 
 		if len(parts) < 1 {
 			continue
@@ -915,13 +934,20 @@ func exists(filename string) bool {
 	return true
 }
 
-func proxyPort(protocol, from, to string, proxy bool) {
-	args := []string{"run", "-p", fmt.Sprintf("%s:%s", from, from), "convox/proxy", from, to, protocol}
+func proxyPort(protocol, from, to, link string, proxy, secure bool) {
+	args := []string{"run", "-p", fmt.Sprintf("%s:%s", from, from), "--link", fmt.Sprintf("%s:host", link), "convox/proxy", from, to, protocol}
 
 	if proxy {
 		args = append(args, "proxy")
 	}
 
+	if secure {
+		args = append(args, "secure")
+	}
+
+	// wait for main container to come up
+	time.Sleep(1 * time.Second)
+
 	cmd := Execer("docker", args...)
 	// cmd.Stdout = os.Stdout
 	// cmd.Stderr = os.Stderr
@@ -1422,15 +1448,15 @@ func detectApplication(dir string) string {
 
 func initApplication(dir string) error {
 	wd, err := os.Getwd()
-	
+
 	if err != nil {
 		return err
 	}
-	
+
 	defer os.Chdir(wd)
-	
+
 	os.Chdir(dir)
-	
+
 	// TODO parse the Dockerfile and build a docker-compose.yml
 	if exists("Dockerfile") || exists("docker-compose.yml") {
 		return nil
@@ -1448,6 +1474,10 @@ func initApplication(dir string) error {
 		return err
 	}
 
+	if err := writeAsset(".dockerignore", fmt.Sprintf("init/%s/.dockerignore", kind)); err != nil {
+		return err
+	}
+
 	return nil
 }
 

api/manifest/manifest_test.go

@@ -55,7 +55,7 @@ func TestPortsWanted(t *testing.T) {
 
 	Init(destDir)
 	m, _ := Read(destDir, defaultManifestFile)
-	ps := m.PortsWanted()
+	ps := m.PortsWanted(0)
 
 	cases := Cases{
 		{ps, []string{"5000"}},
@@ -235,7 +235,7 @@ func testBuild(m *Manifest, app string) (string, string) {
 }
 
 func testRun(m *Manifest, app string) (string, string) {
-	return testRunner(m, app, func() { m.Run(app, true) })
+	return testRunner(m, app, func() { m.Run(app, true, 0) })
 }
 
 func testRunner(m *Manifest, app string, fn runnerFn) (string, string) {

api/models/fixtures/Makefile

@@ -0,0 +1,9 @@
+FIXTURES := $(shell find . -name '*.yml')
+MANIFESTS := $(FIXTURES:%.yml=%.json)
+
+.PHONY: all
+
+all: $(MANIFESTS)
+
+%.json: %.yml
+	env AWS_REGION=us-test-2 PROVIDER=test RACK=convox-test fixture $< > $@

api/models/fixtures/balancer_labels.json

@@ -135,6 +135,16 @@
       "Description": "CPU shares of each process",
       "Type": "Number"
     },
+    "DeploymentMaximum": {
+      "Default": "200",
+      "Description": "Maximum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
+    "DeploymentMinimum": {
+      "Default": "100",
+      "Description": "Minimum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
     "Environment": {
       "Default": "",
       "Description": "",
@@ -513,6 +523,12 @@
         "Cluster": {
           "Ref": "Cluster"
         },
+        "DeploymentMaximumPercent": {
+          "Ref": "DeploymentMaximum"
+        },
+        "DeploymentMinimumPercent": {
+          "Ref": "DeploymentMinimum"
+        },
         "DesiredCount": {
           "Ref": "MainDesiredCount"
         },

api/models/fixtures/command_exec_form.json

@@ -103,6 +103,16 @@
       "Description": "CPU shares of each process",
       "Type": "Number"
     },
+    "DeploymentMaximum": {
+      "Default": "200",
+      "Description": "Maximum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
+    "DeploymentMinimum": {
+      "Default": "100",
+      "Description": "Minimum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
     "Environment": {
       "Default": "",
       "Description": "",
@@ -259,6 +269,12 @@
         "Cluster": {
           "Ref": "Cluster"
         },
+        "DeploymentMaximumPercent": {
+          "Ref": "DeploymentMaximum"
+        },
+        "DeploymentMinimumPercent": {
+          "Ref": "DeploymentMinimum"
+        },
         "DesiredCount": {
           "Ref": "MainDesiredCount"
         },

api/models/fixtures/command_string_form.json

@@ -103,6 +103,16 @@
       "Description": "CPU shares of each process",
       "Type": "Number"
     },
+    "DeploymentMaximum": {
+      "Default": "200",
+      "Description": "Maximum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
+    "DeploymentMinimum": {
+      "Default": "100",
+      "Description": "Minimum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
     "Environment": {
       "Default": "",
       "Description": "",
@@ -259,6 +269,12 @@
         "Cluster": {
           "Ref": "Cluster"
         },
+        "DeploymentMaximumPercent": {
+          "Ref": "DeploymentMaximum"
+        },
+        "DeploymentMinimumPercent": {
+          "Ref": "DeploymentMinimum"
+        },
         "DesiredCount": {
           "Ref": "MainDesiredCount"
         },

api/models/fixtures/complex_environment.json

@@ -103,6 +103,16 @@
       "Description": "CPU shares of each process",
       "Type": "Number"
     },
+    "DeploymentMaximum": {
+      "Default": "200",
+      "Description": "Maximum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
+    "DeploymentMinimum": {
+      "Default": "100",
+      "Description": "Minimum percentage of processes to keep running while deploying",
+      "Type": "Number"
+    },
     "Environment": {
       "Default": "",
       "Description": "",
@@ -259,6 +269,12 @@
         "Cluster": {
           "Ref": "Cluster"
         },
+        "DeploymentMaximumPercent": {
+          "Ref": "DeploymentMaximum"
+        },
+        "DeploymentMinimumPercent": {
+          "Ref": "DeploymentMinimum"
+        },
         "DesiredCount": {
           "Ref": "MainDesiredCount"
         },