refactor: encapsulate existing SDK data system [DRAFT]

config.go

@@ -197,4 +197,41 @@ type Config struct {
 	// LaunchDarkly provides integration packages, and most applications will not
 	// need to implement their own hooks.
 	Hooks []ldhooks.Hook
+
+	// This field is not stable, and not subject to any backwards compatibility guarantees or semantic versioning.
+	// It is not suitable for production usage. Do not use it. You have been warned.
+	//
+	// DataSystem configures how data (e.g. flags, segments) are retrieved by the SDK.
+	//
+	// Set this field only if you want to specify non-default values for any of the data system configuration,
+	// such as defining an alternate data source or setting up a persistent store.
+	//
+	// Below, the default configuration is described with the relevant config item in parentheses:
+	// 1. The SDK will first attempt to fetch all data from LaunchDarkly's global Content Delivery Network (Initializer)
+	// 2. It will then establish a streaming connection with LaunchDarkly's realtime Flag Delivery Network (Primary
+	//    Synchronizer.)
+	// 3. If at any point the connection to the realtime network is interrupted for a short period of time,
+	//    the connection will be automatically re-established.
+	// 4. If the connection cannot be re-established over a sustained period, the SDK will begin to make periodic
+	//    requests to LaunchDarkly's global CDN (Secondary Synchronizer)
+	// 5. After a period of time, the SDK will swap back to the realtime Flag Delivery Network if it becomes
+	//    available again.
+	//
+	// The default streaming mode configuration is preferred for most use-cases (DataSystem().StreamingPreferred()).
+	// Sometimes streaming connections are blocked by firewalls or proxies. If this is the case, a polling-only mode
+	// can be configured:
+	//
+	//    config := ld.Config{
+	//        DataSystem: ldcomponents.DataSystem().PollingOnly(),
+	//    }
+	//
+	// If you'd like to load data from a local source to provide redundancy if there is a problem
+	// connecting to LaunchDarkly, you can add a custom initializer:
+	//
+	//    config := ld.Config {
+	// 	  		DataSystem: ldcomponents.DataSystem().PrependInitializers(myCustomInitializer),
+	//    }
+	//
+	// The initializer(s) will run before LaunchDarkly's default initializer.
+	DataSystem subsystems.ComponentConfigurer[subsystems.DataSystemConfiguration]
 }

internal/datasource/streaming_data_source_events.go

@@ -16,7 +16,7 @@ var (
 	deleteDataRequiredProperties = []string{"path", "version"} //nolint:gochecknoglobals
 )
 
-// This is the logical representation of the data in the "put" event. In the JSON representation,
+// PutData is the logical representation of the data in the "put" event. In the JSON representation,
 // the "data" property is actually a map of maps, but the schema we use internally is a list of
 // lists instead.
 //
@@ -37,12 +37,12 @@ var (
 //	    }
 //	  }
 //	}
-type putData struct {
+type PutData struct {
 	Path string // we don't currently do anything with this
 	Data []ldstoretypes.Collection
 }
 
-// This is the logical representation of the data in the "patch" event. In the JSON representation,
+// PatchData is the logical representation of the data in the "patch" event. In the JSON representation,
 // there is a "path" property in the format "/flags/key" or "/segments/key", which we convert into
 // Kind and Key when we parse it. The "data" property is the JSON representation of the flag or
 // segment, which we deserialize into an ItemDescriptor.
@@ -56,13 +56,13 @@ type putData struct {
 //	    "version": 2, ...etc.
 //	  }
 //	}
-type patchData struct {
+type PatchData struct {
 	Kind ldstoretypes.DataKind
 	Key  string
 	Data ldstoretypes.ItemDescriptor
 }
 
-// This is the logical representation of the data in the "delete" event. In the JSON representation,
+// DeleteData is the logical representation of the data in the "delete" event. In the JSON representation,
 // there is a "path" property in the format "/flags/key" or "/segments/key", which we convert into
 // Kind and Key when we parse it.
 //
@@ -72,14 +72,14 @@ type patchData struct {
 //	  "path": "/flags/flagkey",
 //	  "version": 3
 //	}
-type deleteData struct {
+type DeleteData struct {
 	Kind    ldstoretypes.DataKind
 	Key     string
 	Version int
 }
 
-func parsePutData(data []byte) (putData, error) {
-	var ret putData
+func parsePutData(data []byte) (PutData, error) {
+	var ret PutData
 	r := jreader.NewReader(data)
 	for obj := r.Object().WithRequiredProperties(putDataRequiredProperties); obj.Next(); {
 		switch string(obj.Name()) {
@@ -92,15 +92,15 @@ func parsePutData(data []byte) (putData, error) {
 	return ret, r.Error()
 }
 
-func parsePatchData(data []byte) (patchData, error) {
-	var ret patchData
+func parsePatchData(data []byte) (PatchData, error) {
+	var ret PatchData
 	r := jreader.NewReader(data)
 	var kind datakinds.DataKindInternal
 	var key string
-	parseItem := func() (patchData, error) {
+	parseItem := func() (PatchData, error) {
 		item, err := kind.DeserializeFromJSONReader(&r)
 		if err != nil {
-			return patchData{}, err
+			return PatchData{}, err
 		}
 		ret.Data = item
 		return ret, nil
@@ -126,7 +126,7 @@ func parsePatchData(data []byte) (patchData, error) {
 		}
 	}
 	if err := r.Error(); err != nil {
-		return patchData{}, err
+		return PatchData{}, err
 	}
 	// If we got here, it means we couldn't parse the data model object yet because we saw the
 	// "data" property first. But we definitely saw both properties (otherwise we would've got
@@ -138,13 +138,13 @@ func parsePatchData(data []byte) (patchData, error) {
 		}
 	}
 	if r.Error() != nil {
-		return patchData{}, r.Error()
+		return PatchData{}, r.Error()
 	}
-	return patchData{}, errors.New("patch event had no data property")
+	return PatchData{}, errors.New("patch event had no data property")
 }
 
-func parseDeleteData(data []byte) (deleteData, error) {
-	var ret deleteData
+func parseDeleteData(data []byte) (DeleteData, error) {
+	var ret DeleteData
 	r := jreader.NewReader(data)
 	for obj := r.Object().WithRequiredProperties(deleteDataRequiredProperties); obj.Next(); {
 		switch string(obj.Name()) {
@@ -161,7 +161,7 @@ func parseDeleteData(data []byte) (deleteData, error) {
 		}
 	}
 	if r.Error() != nil {
-		return deleteData{}, r.Error()
+		return DeleteData{}, r.Error()
 	}
 	return ret, nil
 }

internal/datasourcev2/helpers.go

@@ -0,0 +1,85 @@
+package datasourcev2
+
+//nolint: godox
+// TODO: This was copied from datasource/helpers.go. We should extract these
+// out into a common module, or if we decide we don't need these later in the
+// v2 implementation, we should clean this up.
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/launchdarkly/go-sdk-common/v3/ldlog"
+)
+
+type httpStatusError struct {
+	Message string
+	Code    int
+}
+
+func (e httpStatusError) Error() string {
+	return e.Message
+}
+
+// Tests whether an HTTP error status represents a condition that might resolve on its own if we retry,
+// or at least should not make us permanently stop sending requests.
+func isHTTPErrorRecoverable(statusCode int) bool {
+	if statusCode >= 400 && statusCode < 500 {
+		switch statusCode {
+		case 400: // bad request
+			return true
+		case 408: // request timeout
+			return true
+		case 429: // too many requests
+			return true
+		default:
+			return false // all other 4xx errors are unrecoverable
+		}
+	}
+	return true
+}
+
+func httpErrorDescription(statusCode int) string {
+	message := ""
+	if statusCode == 401 || statusCode == 403 {
+		message = " (invalid SDK key)"
+	}
+	return fmt.Sprintf("HTTP error %d%s", statusCode, message)
+}
+
+// Logs an HTTP error or network error at the appropriate level and determines whether it is recoverable
+// (as defined by isHTTPErrorRecoverable).
+func checkIfErrorIsRecoverableAndLog(
+	loggers ldlog.Loggers,
+	errorDesc, errorContext string,
+	statusCode int,
+	recoverableMessage string,
+) bool {
+	if statusCode > 0 && !isHTTPErrorRecoverable(statusCode) {
+		loggers.Errorf("Error %s (giving up permanently): %s", errorContext, errorDesc)
+		return false
+	}
+	loggers.Warnf("Error %s (%s): %s", errorContext, recoverableMessage, errorDesc)
+	return true
+}
+
+func checkForHTTPError(statusCode int, url string) error {
+	if statusCode == http.StatusUnauthorized {
+		return httpStatusError{
+			Message: fmt.Sprintf("Invalid SDK key when accessing URL: %s. Verify that your SDK key is correct.", url),
+			Code:    statusCode}
+	}
+
+	if statusCode == http.StatusNotFound {
+		return httpStatusError{
+			Message: fmt.Sprintf("Resource not found when accessing URL: %s. Verify that this resource exists.", url),
+			Code:    statusCode}
+	}
+
+	if statusCode/100 != 2 {
+		return httpStatusError{
+			Message: fmt.Sprintf("Unexpected response code: %d when accessing URL: %s", statusCode, url),
+			Code:    statusCode}
+	}
+	return nil
+}

internal/datasourcev2/package_info.go

@@ -0,0 +1,9 @@
+// Package datasourcev2 is an internal package containing implementation types for the SDK's data source
+// implementations (streaming, polling, etc.) and related functionality. These types are not visible
+// from outside of the SDK.
+//
+// WARNING: This particular implementation supports the upcoming flag delivery v2 format which is not
+// publicly available.
+//
+// This does not include the file data source, which is in the ldfiledata package.
+package datasourcev2