Enable extended resource definitions for scheduling purposes.

proxy/deploy.go

@@ -90,29 +90,36 @@ func Deploy(fprocess string, gateway string, functionName string, image string,
 
 	hasLimits := false
 	req.Limits = &requests.FunctionResources{}
-	if functionResourceRequest1.Limits != nil && len(functionResourceRequest1.Limits.Memory) > 0 {
+	if functionResourceRequest1.Limits != nil {
 		hasLimits = true
-		req.Limits.Memory = functionResourceRequest1.Limits.Memory
-	}
-	if functionResourceRequest1.Limits != nil && len(functionResourceRequest1.Limits.CPU) > 0 {
-		hasLimits = true
-		req.Limits.CPU = functionResourceRequest1.Limits.CPU
+		if len(functionResourceRequest1.Limits.Memory) > 0 {
+			req.Limits.Memory = functionResourceRequest1.Limits.Memory
+		}
+		if len(functionResourceRequest1.Limits.CPU) > 0 {
+			req.Limits.CPU = functionResourceRequest1.Limits.CPU
+		}
+		if len(functionResourceRequest1.Limits.Others) > 0 {
+			req.Limits.Others = functionResourceRequest1.Limits.Others
+		}
 	}
 	if !hasLimits {
 		req.Limits = nil
 	}
 
 	hasRequests := false
 	req.Requests = &requests.FunctionResources{}
-	if functionResourceRequest1.Requests != nil && len(functionResourceRequest1.Requests.Memory) > 0 {
+	if functionResourceRequest1.Requests != nil {
 		hasRequests = true
-		req.Requests.Memory = functionResourceRequest1.Requests.Memory
-	}
-	if functionResourceRequest1.Requests != nil && len(functionResourceRequest1.Requests.CPU) > 0 {
-		hasRequests = true
-		req.Requests.CPU = functionResourceRequest1.Requests.CPU
+		if len(functionResourceRequest1.Requests.Memory) > 0 {
+			req.Requests.Memory = functionResourceRequest1.Requests.Memory
+		}
+		if len(functionResourceRequest1.Requests.CPU) > 0 {
+			req.Requests.CPU = functionResourceRequest1.Requests.CPU
+		}
+		if len(functionResourceRequest1.Requests.Others) > 0 {
+			req.Requests.Others = functionResourceRequest1.Requests.Others
+		}
 	}
-
 	if !hasRequests {
 		req.Requests = nil
 	}

stack/schema.go

@@ -58,10 +58,11 @@ type Function struct {
 	Annotations *map[string]string `yaml:"annotations"`
 }
 
-// FunctionResources Memory and CPU
+// FunctionResources Memory, CPU and other extended resources
 type FunctionResources struct {
 	Memory string `yaml:"memory"`
 	CPU    string `yaml:"cpu"`
+	Others map[string]string
 }
 
 // EnvironmentFile represents external file for environment data

stack/stack.go

@@ -5,21 +5,20 @@ package stack
 
 import (
 	"fmt"
+	"github.com/ryanuber/go-glob"
+	yaml "gopkg.in/yaml.v2"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
 	"regexp"
 	"time"
-
-	"github.com/ryanuber/go-glob"
-	yaml "gopkg.in/yaml.v2"
 )
 
 const providerName = "faas"
 const providerNameLong = "openfaas"
 
-// ParseYAMLData parse YAML file into a stack of "services".
+// ParseYAMLFile parse YAML file into a stack of "services".
 func ParseYAMLFile(yamlFile, regex, filter string) (*Services, error) {
 	var err error
 	var fileData []byte
@@ -94,6 +93,33 @@ func ParseYAMLData(fileData []byte, regex string, filter string) (*Services, err
 	return &services, nil
 }
 
+// Custom unmarshaler to allow scheduling of extended resources such as GPUs, FPGAs from various vendors
+func (config *FunctionResources) UnmarshalYAML(unmarshal func(interface{}) error) error {
+
+	var raw map[string]string
+	var others = make(map[string]string)
+
+	r, _ := regexp.Compile("^[[:graph:]]+[.]{1}[[:alnum:]]+/(gpu|fpga)$")
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+	for key, value := range raw {
+		switch {
+		case key == "cpu":
+			config.CPU = value
+		case key == "memory":
+			config.Memory = value
+		case r.MatchString(key):
+			others[key] = value
+		default:
+			fmt.Errorf("Ignoring unknown extended resource: %s with value: %s\n", key, value)
+		}
+	}
+	config.Others = others
+	return nil
+}
+
 func makeHTTPClient(timeout *time.Duration) http.Client {
 	if timeout != nil {
 		return http.Client{

stack/stack_test.go

@@ -42,12 +42,79 @@ functions:
     handler: ./sample/abcd-eeee
     image: stuff2/stuff23423
 `
-
 const TestData_2 string = `provider:
   name: faas
   gateway: http://127.0.0.1:8080
   network: "func_functions"
+`
+const TestData_ExtResources string = `provider:
+  name: faas
+  gateway: http://127.0.0.1:8080
 
+functions:
+  f1:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      cpu: 0.1
+      vendor.domain/gpu: 1
+  f2:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      memory: 10m
+      vendor.domain/fpga: 1
+  f3:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      cpu: 0.1
+      memory: 10m
+      vendor.domain/gpu: 1
+      vendor.domain/fpga: 1
+  f4:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      vendor_1.domain/gpu: 1
+      vendor_2.domain/gpu: 1
+      vendor_3.domain/fpga: 1
+  f5:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      something/gpu: 1
+  f6:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      something/fpga: 1
+  f7:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      some.vendor/gpu: 1
+      some.vendor/fastgpu: 1
+  f8:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      some.vendor/fpga: 1
+      some.vendor/fastfpga: 1
+  f9:
+    lang: node
+    handler: handler
+    image: image
+    limits:
+      random: 1
 `
 
 const noMatchesErrorMsg string = "no functions matching --filter/--regex were found in the YAML file"
@@ -375,3 +442,93 @@ func Test_ParseYAMLData_ProviderValues(t *testing.T) {
 		})
 	}
 }
+
+var ParseYAMLTests_ExtResources = []struct {
+	title    string
+	function string
+	expected []string
+}{
+	{
+		title:    "Valid resource: gpu",
+		function: "f1",
+		expected: []string{"vendor.domain/gpu"},
+	},
+	{
+		title:    "Valid resource: fpga",
+		function: "f2",
+		expected: []string{"vendor.domain/fpga"},
+	},
+	{
+		title:    "Valid resources: gpu, fpga",
+		function: "f3",
+		expected: []string{"vendor.domain/fpga", "vendor.domain/gpu"},
+	},
+	{
+		title:    "Valid resources: gpu, gpu, fpga",
+		function: "f4",
+		expected: []string{"vendor_1.domain/gpu", "vendor_2.domain/gpu", "vendor_3.domain/fpga"},
+	},
+	{
+		title:    "Resource specified with invalid domain: something/gpu",
+		function: "f5",
+		expected: []string{},
+	},
+	{
+		title:    "Resource specified with invalid domain: something/fpga",
+		function: "f6",
+		expected: []string{},
+	},
+	{
+		title:    "Invalid resource: fastgpu",
+		function: "f7",
+		expected: []string{"some.vendor/gpu"},
+	},
+	{
+		title:    "Invalid resource: fastfpga",
+		function: "f8",
+		expected: []string{"some.vendor/fpga"},
+	},
+	{
+		title:    "Invalid resource: random",
+		function: "f9",
+		expected: []string{},
+	},
+}
+
+/**
+ * Test parsing of extended resources, such as GPUs and FPGAs
+ */
+func Test_ParseYAMLData_ExtResources(t *testing.T) {
+
+	for _, test := range ParseYAMLTests_ExtResources {
+		t.Run(test.title, func(t *testing.T) {
+			parsedYAML, err := ParseYAMLData([]byte(TestData_ExtResources), test.function, "")
+
+			if err != nil {
+				t.Errorf("Test_ParseYAMLData_ExtResources [%s] test failed: %v", test.title, err)
+				return
+			}
+
+			for _, function := range parsedYAML.Functions {
+				keys := reflect.ValueOf(function.Limits.Others).MapKeys()
+
+				parsed := make([]string, len(keys))
+
+				for i := 0; i < len(keys); i++ {
+					parsed[i] = keys[i].String()
+				}
+				sort.Strings(parsed)
+
+				if !reflect.DeepEqual(parsed, test.expected) {
+					t.Errorf("Test_ParseYAMLData_ExtResources [%s] test failed, does not match expected result;\n  parsed resources:   [%v]\n  expected resources: [%v]",
+						test.title,
+						parsed,
+						test.expected,
+					)
+				}
+
+				t.Log(parsed)
+			}
+		})
+	}
+}