Add http api for fetching accounts

config/config_test.go

@@ -652,8 +652,7 @@ func TestValidateAccountsConfigRestrictions(t *testing.T) {
 	cfg.Accounts.Postgres.ConnectionInfo.Database = "accounts"
 
 	errs := cfg.validate()
-	assert.Len(t, errs, 2)
-	assert.Contains(t, errs, errors.New("accounts.http: retrieving accounts via http not available, use accounts.files"))
+	assert.Len(t, errs, 1)
 	assert.Contains(t, errs, errors.New("accounts.postgres: retrieving accounts via postgres not available, use accounts.files"))
 }
 

config/stored_requests.go

@@ -137,9 +137,6 @@ func resolvedStoredRequestsConfig(cfg *Configuration) {
 }
 
 func (cfg *StoredRequests) validate(errs configErrors) configErrors {
-	if cfg.DataType() == AccountDataType && cfg.HTTP.Endpoint != "" {
-		errs = append(errs, fmt.Errorf("%s.http: retrieving accounts via http not available, use accounts.files", cfg.Section()))
-	}
 	if cfg.DataType() == AccountDataType && cfg.Postgres.ConnectionInfo.Database != "" {
 		errs = append(errs, fmt.Errorf("%s.postgres: retrieving accounts via postgres not available, use accounts.files", cfg.Section()))
 	} else {

stored_requests/backends/http_fetcher/fetcher.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/prebid/prebid-server/stored_requests"
@@ -19,9 +20,13 @@ import (
 //
 // This file expects the endpoint to satisfy the following API:
 //
+// Stored requests
 // GET {endpoint}?request-ids=["req1","req2"]&imp-ids=["imp1","imp2","imp3"]
 //
-// This endpoint should return a payload like:
+// Accounts
+// GET {endpoint}?account-ids=["acc1","acc2"]
+//
+// The above endpoints should return a payload like:
 //
 // {
 //   "requests": {
@@ -34,21 +39,33 @@ import (
 //     "imp3": null // If imp3 is not found
 //   }
 // }
+// or
+// {
+//   "accounts": {
+//     "acc1": { ... config data for acc1 ... },
+//     "acc2": { ... config data for acc2 ... },
+//   },
+// }
 //
 //
 func NewFetcher(client *http.Client, endpoint string) *HttpFetcher {
 	// Do some work up-front to figure out if the (configurable) endpoint has a query string or not.
 	// When we build requests, we'll either want to add `?request-ids=...&imp-ids=...` _or_
-	// `&request-ids=...&imp-ids=...`, depending.
+	// `&request-ids=...&imp-ids=...`.
+
+	if url, err := url.Parse(endpoint); err != nil {
+		glog.Errorf(`Invalid endpoint "%s": %v`, endpoint, err)
+	} else {
+		glog.Infof("Making http_fetcher for endpoint %v", url)
+	}
+
 	urlPrefix := endpoint
 	if strings.Contains(endpoint, "?") {
 		urlPrefix = urlPrefix + "&"
 	} else {
 		urlPrefix = urlPrefix + "?"
 	}
 
-	glog.Info("Making http_fetcher which calls GET " + urlPrefix + "request-ids=%REQUEST_ID_LIST%&imp-ids=%IMP_ID_LIST%")
-
 	return &HttpFetcher{
 		client:   client,
 		Endpoint: urlPrefix,
@@ -81,8 +98,68 @@ func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []stri
 	return
 }
 
-func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
-	return nil, []error{stored_requests.NotFoundError{accountID, "Account"}}
+// FetchAccounts retrieves account configurations
+//
+// Request format is similar to the one for requests:
+// GET {endpoint}?account-ids=["account1","account2",...]
+//
+// The endpoint is expected to respond with a JSON map with accountID -> json.RawMessage
+// {
+//   "account1": { ... account json ... }
+// }
+// The JSON contents of account config is returned as-is (NOT validated)
+func (fetcher *HttpFetcher) FetchAccounts(ctx context.Context, accountIDs []string) (map[string]json.RawMessage, []error) {
+	if len(accountIDs) == 0 {
+		return nil, nil
+	}
+	httpReq, err := http.NewRequestWithContext(ctx, "GET", fetcher.Endpoint+"account-ids=[\""+strings.Join(accountIDs, "\",\"")+"\"]", nil)
+	if err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: build request failed with %v`, accountIDs, err),
+		}
+	}
+	httpResp, err := ctxhttp.Do(ctx, fetcher.client, httpReq)
+	if err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: %v`, accountIDs, err),
+		}
+	}
+	defer httpResp.Body.Close()
+	respBytes, err := ioutil.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: error reading response: %v`, accountIDs, err),
+		}
+	}
+	if httpResp.StatusCode != http.StatusOK {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: unexpected response status %d`, accountIDs, httpResp.StatusCode),
+		}
+	}
+	var responseData accountsResponseContract
+	if err = json.Unmarshal(respBytes, &responseData); err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: failed to parse response: %v`, accountIDs, err),
+		}
+	}
+	errs := convertNullsToErrs(responseData.Accounts, "Account", nil)
+	return responseData.Accounts, errs
+}
+
+// FetchAccount fetchers a single accountID and returns its corresponding json
+func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string) (accountJSON json.RawMessage, errs []error) {
+	accountData, errs := fetcher.FetchAccounts(ctx, []string{accountID})
+	if len(errs) > 0 {
+		return nil, errs
+	}
+	accountJSON, ok := accountData[accountID]
+	if !ok {
+		return nil, []error{stored_requests.NotFoundError{
+			ID:       accountID,
+			DataType: "Account",
+		}}
+	}
+	return accountJSON, nil
 }
 
 func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
@@ -190,3 +267,7 @@ type responseContract struct {
 	Requests map[string]json.RawMessage `json:"requests"`
 	Imps     map[string]json.RawMessage `json:"imps"`
 }
+
+type accountsResponseContract struct {
+	Accounts map[string]json.RawMessage `json:"accounts"`
+}

stored_requests/backends/http_fetcher/fetcher_test.go

@@ -9,6 +9,8 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestSingleReq(t *testing.T) {
@@ -17,8 +19,8 @@ func TestSingleReq(t *testing.T) {
 
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), []string{"req-1"}, nil)
 	assertMapKeys(t, reqData, "req-1")
-	assertMapKeys(t, impData)
-	assertErrLength(t, errs, 0)
+	assert.Empty(t, impData)
+	assert.Empty(t, errs)
 }
 
 func TestSeveralReqs(t *testing.T) {
@@ -27,28 +29,28 @@ func TestSeveralReqs(t *testing.T) {
 
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), []string{"req-1", "req-2"}, nil)
 	assertMapKeys(t, reqData, "req-1", "req-2")
-	assertMapKeys(t, impData)
-	assertErrLength(t, errs, 0)
+	assert.Empty(t, impData)
+	assert.Empty(t, errs)
 }
 
 func TestSingleImp(t *testing.T) {
 	fetcher, close := newTestFetcher(t, nil, []string{"imp-1"})
 	defer close()
 
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), nil, []string{"imp-1"})
-	assertMapKeys(t, reqData)
+	assert.Empty(t, reqData)
 	assertMapKeys(t, impData, "imp-1")
-	assertErrLength(t, errs, 0)
+	assert.Empty(t, errs)
 }
 
 func TestSeveralImps(t *testing.T) {
 	fetcher, close := newTestFetcher(t, nil, []string{"imp-1", "imp-2"})
 	defer close()
 
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), nil, []string{"imp-1", "imp-2"})
-	assertMapKeys(t, reqData)
+	assert.Empty(t, reqData)
 	assertMapKeys(t, impData, "imp-1", "imp-2")
-	assertErrLength(t, errs, 0)
+	assert.Empty(t, errs)
 }
 
 func TestReqsAndImps(t *testing.T) {
@@ -58,17 +60,62 @@ func TestReqsAndImps(t *testing.T) {
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), []string{"req-1"}, []string{"imp-1"})
 	assertMapKeys(t, reqData, "req-1")
 	assertMapKeys(t, impData, "imp-1")
-	assertErrLength(t, errs, 0)
+	assert.Empty(t, errs)
 }
 
 func TestMissingValues(t *testing.T) {
 	fetcher, close := newEmptyFetcher(t, []string{"req-1", "req-2"}, []string{"imp-1"})
 	defer close()
 
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), []string{"req-1", "req-2"}, []string{"imp-1"})
-	assertMapKeys(t, reqData)
-	assertMapKeys(t, impData)
-	assertErrLength(t, errs, 3)
+	assert.Empty(t, reqData)
+	assert.Empty(t, impData)
+	assert.Len(t, errs, 3)
+}
+
+func TestFetchAccounts(t *testing.T) {
+	fetcher, close := newTestAccountFetcher(t, []string{"acc-1", "acc-2"})
+	defer close()
+
+	accData, errs := fetcher.FetchAccounts(context.Background(), []string{"acc-1", "acc-2"})
+	assert.Empty(t, errs)
+	assertMapKeys(t, accData, "acc-1", "acc-2")
+}
+
+func TestFetchAccountsNoData(t *testing.T) {
+	fetcher, close := newFetcherBrokenBackend()
+	defer close()
+
+	accData, errs := fetcher.FetchAccounts(context.Background(), []string{"req-1"})
+	assert.Len(t, errs, 1)
+	assert.Nil(t, accData)
+}
+
+func TestFetchAccountsBadJSON(t *testing.T) {
+	fetcher, close := newFetcherBadJSON()
+	defer close()
+
+	accData, errs := fetcher.FetchAccounts(context.Background(), []string{"req-1"})
+	assert.Len(t, errs, 1)
+	assert.Nil(t, accData)
+}
+
+func TestFetchAccount(t *testing.T) {
+	fetcher, close := newTestAccountFetcher(t, []string{"acc-1"})
+	defer close()
+
+	account, errs := fetcher.FetchAccount(context.Background(), "acc-1")
+	assert.Empty(t, errs, "Unexpected error fetching existing account")
+	assert.JSONEq(t, `"acc-1"`, string(account), "Unexpected account data returned")
+}
+
+func TestFetchAccountNoData(t *testing.T) {
+	fetcher, close := newFetcherBrokenBackend()
+	defer close()
+
+	unknownAccount, errs := fetcher.FetchAccount(context.Background(), "unknown-acc")
+	assert.NotEmpty(t, errs, "Retrieving unknown account should have returned an error")
+	assert.Nil(t, unknownAccount)
 }
 
 func TestErrResponse(t *testing.T) {
@@ -77,7 +124,7 @@ func TestErrResponse(t *testing.T) {
 	reqData, impData, errs := fetcher.FetchRequests(context.Background(), []string{"req-1"}, []string{"imp-1"})
 	assertMapKeys(t, reqData)
 	assertMapKeys(t, impData)
-	assertErrLength(t, errs, 1)
+	assert.Len(t, errs, 1)
 }
 
 func assertSameContents(t *testing.T, expected map[string]json.RawMessage, actual map[string]json.RawMessage) {
@@ -124,6 +171,14 @@ func newFetcherBrokenBackend() (fetcher *HttpFetcher, closer func()) {
 	return NewFetcher(server.Client(), server.URL), server.Close
 }
 
+func newFetcherBadJSON() (fetcher *HttpFetcher, closer func()) {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`broken JSON`))
+	}
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	return NewFetcher(server.Client(), server.URL), server.Close
+}
+
 func newEmptyFetcher(t *testing.T, expectReqIDs []string, expectImpIDs []string) (fetcher *HttpFetcher, closer func()) {
 	handler := newHandler(t, expectReqIDs, expectImpIDs, jsonifyToNull)
 	server := httptest.NewServer(http.HandlerFunc(handler))
@@ -139,12 +194,12 @@ func newTestFetcher(t *testing.T, expectReqIDs []string, expectImpIDs []string)
 func newHandler(t *testing.T, expectReqIDs []string, expectImpIDs []string, jsonifier func(string) json.RawMessage) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		query := r.URL.Query()
-		assertMatches(t, query.Get("request-ids"), expectReqIDs)
-		assertMatches(t, query.Get("imp-ids"), expectImpIDs)
-
 		gotReqIDs := richSplit(query.Get("request-ids"))
 		gotImpIDs := richSplit(query.Get("imp-ids"))
 
+		assertMatches(t, gotReqIDs, expectReqIDs)
+		assertMatches(t, gotImpIDs, expectImpIDs)
+
 		reqIDResponse := make(map[string]json.RawMessage, len(gotReqIDs))
 		impIDResponse := make(map[string]json.RawMessage, len(gotImpIDs))
 
@@ -174,10 +229,43 @@ func newHandler(t *testing.T, expectReqIDs []string, expectImpIDs []string, json
 	}
 }
 
-func assertMatches(t *testing.T, query string, expected []string) {
+func newTestAccountFetcher(t *testing.T, expectAccIDs []string) (fetcher *HttpFetcher, closer func()) {
+	handler := newAccountHandler(t, expectAccIDs, jsonifyID)
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	return NewFetcher(server.Client(), server.URL), server.Close
+}
+
+func newAccountHandler(t *testing.T, expectAccIDs []string, jsonifier func(string) json.RawMessage) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		query := r.URL.Query()
+		gotAccIDs := richSplit(query.Get("account-ids"))
+
+		assertMatches(t, gotAccIDs, expectAccIDs)
+
+		accIDResponse := make(map[string]json.RawMessage, len(gotAccIDs))
+
+		for _, accID := range gotAccIDs {
+			if accID != "" {
+				accIDResponse[accID] = jsonifier(accID)
+			}
+		}
+
+		respObj := accountsResponseContract{
+			Accounts: accIDResponse,
+		}
+
+		if respBytes, err := json.Marshal(respObj); err != nil {
+			t.Errorf("failed to marshal responseContract in test:  %v", err)
+			w.WriteHeader(http.StatusInternalServerError)
+		} else {
+			w.Write(respBytes)
+		}
+	}
+}
+
+func assertMatches(t *testing.T, queryVals []string, expected []string) {
 	t.Helper()
 
-	queryVals := richSplit(query)
 	if len(queryVals) == 1 && queryVals[0] == "" {
 		if len(expected) != 0 {
 			t.Errorf("Expected no query vals, but got %v", queryVals)
@@ -250,11 +338,3 @@ func assertMapKeys(t *testing.T, m map[string]json.RawMessage, keys ...string) {
 		}
 	}
 }
-
-func assertErrLength(t *testing.T, errs []error, expected int) {
-	t.Helper()
-
-	if len(errs) != expected {
-		t.Errorf("Expected %d errors. Got: %v", expected, errs)
-	}
-}