Merge pull request #357 from hairyhenderson/dynamic-datasource

Allowing datasources to be defined dynamically

data/datasource.go

@@ -263,11 +263,28 @@ func (d *Data) DatasourceExists(alias string) bool {
 	return ok
 }
 
-// Datasource -
-func (d *Data) Datasource(alias string, args ...string) (interface{}, error) {
+func (d *Data) lookupSource(alias string) (*Source, error) {
 	source, ok := d.Sources[alias]
 	if !ok {
-		return nil, errors.Errorf("Undefined datasource '%s'", alias)
+		srcURL, err := url.Parse(alias)
+		if err != nil || !srcURL.IsAbs() {
+			return nil, errors.Errorf("Undefined datasource '%s'", alias)
+		}
+		source, err = NewSource(alias, srcURL)
+		if err != nil {
+			return nil, err
+		}
+		source.Header = d.extraHeaders[alias]
+		d.Sources[alias] = source
+	}
+	return source, nil
+}
+
+// Datasource -
+func (d *Data) Datasource(alias string, args ...string) (interface{}, error) {
+	source, err := d.lookupSource(alias)
+	if err != nil {
+		return nil, err
 	}
 	b, err := d.ReadSource(source, args...)
 	if err != nil {

data/datasource_test.go

@@ -168,6 +168,9 @@ func TestDatasource(t *testing.T) {
 	actual, err := d.Datasource("foo")
 	assert.NoError(t, err)
 	assert.Equal(t, "", actual)
+
+	_, err = d.Datasource("bar")
+	assert.Error(t, err)
 }
 
 func TestDatasourceReachable(t *testing.T) {
@@ -247,12 +250,18 @@ func TestHTTPFile(t *testing.T) {
 	data := &Data{
 		Sources: sources,
 	}
-	expected := make(map[string]interface{})
-	expected["hello"] = "world"
-	d, err := data.Datasource("foo")
+
+	expected := map[string]interface{}{
+		"hello": "world",
+	}
+
+	actual, err := data.Datasource("foo")
+	assert.NoError(t, err)
+	assert.Equal(t, marshalObj(expected, json.Marshal), marshalObj(actual, json.Marshal))
+
+	actual, err = data.Datasource(server.URL)
 	assert.NoError(t, err)
-	actual := d.(map[string]interface{})
-	assert.Equal(t, expected["hello"], actual["hello"])
+	assert.Equal(t, marshalObj(expected, json.Marshal), marshalObj(actual, json.Marshal))
 }
 
 func TestHTTPFileWithHeaders(t *testing.T) {
@@ -285,6 +294,19 @@ func TestHTTPFileWithHeaders(t *testing.T) {
 	actual, err := data.Datasource("foo")
 	assert.NoError(t, err)
 	assert.Equal(t, marshalObj(expected, json.Marshal), marshalObj(actual, json.Marshal))
+
+	expected = http.Header{
+		"Accept-Encoding": {"test"},
+		"Foo":             {"bar", "baz"},
+		"User-Agent":      {"Go-http-client/1.1"},
+	}
+	data = &Data{
+		Sources:      sources,
+		extraHeaders: map[string]http.Header{server.URL: expected},
+	}
+	actual, err = data.Datasource(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, marshalObj(expected, json.Marshal), marshalObj(actual, json.Marshal))
 }
 
 func TestParseHeaderArgs(t *testing.T) {

docs/content/functions/data.md

@@ -9,7 +9,9 @@ A collection of functions that retrieve, parse, and convert structured data.
 
 ## `datasource`
 
-Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument).
+Parses a given datasource (provided by the [`--datasource/-d`](#--datasource-d) argument or [`defineDatasource`](#definedatasource)).
+
+If the `alias` is undefined, but is a valid URL, `datasource` will dynamically read from that URL. 
 
 See [Datasources](../../datasources) for (much!) more information.
 
@@ -23,7 +25,7 @@ datasource alias [subpath]
 
 | name   | description |
 |--------|-------|
-| `alias` | the datasource alias, as provided by [`--datasource`/`-d`](../usage/#datasource-d) |
+| `alias` | the datasource alias (or a URL for dynamic use) |
 | `subpath` | _(optional)_ the subpath to use, if supported by the datasource |
 
 ### Examples

test/integration/datasources_http_test.go

@@ -42,4 +42,8 @@ func (s *DatasourcesHTTPSuite) TestReportsVersion(c *C) {
 		"-H", "foo=Foo:bar",
 		"-i", "{{defineDatasource `foo` `http://"+s.l.Addr().String()+"/`}}{{ index (ds `foo`).headers.Foo 0 }}")
 	result.Assert(c, icmd.Expected{ExitCode: 0, Out: "bar"})
+
+	result = icmd.RunCommand(GomplateBin,
+		"-i", "{{ $d := ds `http://"+s.l.Addr().String()+"/`}}{{ index (index $d.headers `Accept-Encoding`) 0 }}")
+	result.Assert(c, icmd.Expected{ExitCode: 0, Out: "gzip"})
 }