feat: payload filtering [v8] (#356)

Adds Payload Filtering support via static or auto-configuration. With Payload Filtering, Relay is able to proxy both default and filtered environments to downstream SDK clients.

config/config.go

@@ -60,6 +60,10 @@ const (
 	// For instance, if EnvDataStorePrefix is "LD-$CID", the value of that setting for an environment
 	// whose ID is "12345" would be "LD-12345".
 	//
+	// If the environment is scoped to a Payload Filter, then the filter key will be concatenated as follows:
+	// Given: "LD-$CID", environment ID "12345" and filter key "microservice-a"
+	// The substituted result would be: "LD-12345.microservice-a"
+	//
 	// The same convention is used in OfflineModeConfig.
 	AutoConfigEnvironmentIDPlaceholder = "$CID"
 )
@@ -106,6 +110,7 @@ type Config struct {
 	Consul      ConsulConfig
 	DynamoDB    DynamoDBConfig
 	Environment map[string]*EnvConfig
+	Filters     map[string]*FiltersConfig
 	Proxy       ProxyConfig
 
 	// Optional configuration for metrics integrations. Note that unlike the other fields in Config,
@@ -244,6 +249,12 @@ type EnvConfig struct {
 	SecureMode    bool             `conf:"LD_SECURE_MODE_"`
 	LogLevel      OptLogLevel      `conf:"LD_LOG_LEVEL_"`
 	TTL           ct.OptDuration   `conf:"LD_TTL_"`
+	ProjKey       string           `conf:"LD_PROJ_KEY_"`
+	FilterKey     FilterKey        // injected based on [filters] section
+}
+
+type FiltersConfig struct {
+	Keys ct.OptStringList `conf:"LD_FILTER_KEYS_"`
 }
 
 // ProxyConfig represents all the supported proxy options.

config/config_field_types.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/launchdarkly/ld-relay/v8/internal/credential"
+
 	"github.com/launchdarkly/go-sdk-common/v3/ldlog"
 )
 
@@ -31,38 +33,102 @@ type EnvironmentID string
 // AutoConfigKey is a type tag to indicate when a string is used as an auto-configuration key.
 type AutoConfigKey string
 
-// SDKCredential is implemented by types that represent an SDK authorization credential (SDKKey, etc.).
-type SDKCredential interface {
-	// GetAuthorizationHeaderValue returns the value that should be passed in an HTTP Authorization header
-	// when using this credential, or "" if the header is not used.
-	GetAuthorizationHeaderValue() string
-}
+// FilterID represents the unique ID for a filter. It is different from the key, which is scoped to the project
+// level.
+type FilterID string
+
+// FilterKey represents the key that should be used when making requests to LaunchDarkly in order to obtain
+// a filtered environment.
+type FilterKey string
+
+// DefaultFilter represents the lack of a filter, meaning a full LaunchDarkly environment.
+const DefaultFilter = FilterKey("")
 
 // GetAuthorizationHeaderValue for SDKKey returns the same string, since SDK keys are passed in
 // the Authorization header.
 func (k SDKKey) GetAuthorizationHeaderValue() string {
 	return string(k)
 }
 
+func (k SDKKey) Defined() bool {
+	return k != ""
+}
+
+func (k SDKKey) String() string {
+	return string(k)
+}
+
+func (k SDKKey) Compare(cr credential.AutoConfig) (credential.SDKCredential, credential.Status) {
+	if cr.SDKKey == k {
+		return nil, credential.Unchanged
+	}
+	if cr.ExpiringSDKKey == k {
+		// If the AutoConfig update contains an ExpiringSDKKey that is equal to *this* key, then it means
+		// this key is now considered deprecated.
+		return cr.SDKKey, credential.Deprecated
+	} else {
+		// Otherwise if the AutoConfig update contains *some other* key, then it means this one must be considered
+		// expired.
+		return cr.SDKKey, credential.Expired
+	}
+}
+
 // GetAuthorizationHeaderValue for MobileKey returns the same string, since mobile keys are passed in the
 // Authorization header.
 func (k MobileKey) GetAuthorizationHeaderValue() string {
 	return string(k)
 }
 
+func (k MobileKey) Defined() bool {
+	return k != ""
+}
+
+func (k MobileKey) String() string {
+	return string(k)
+}
+
+func (k MobileKey) Compare(cr credential.AutoConfig) (credential.SDKCredential, credential.Status) {
+	if cr.MobileKey == k {
+		return nil, credential.Unchanged
+	}
+	return cr.MobileKey, credential.Expired
+}
+
 // GetAuthorizationHeaderValue for EnvironmentID returns an empty string, since environment IDs are not
 // passed in a header but as part of the request URL.
 func (k EnvironmentID) GetAuthorizationHeaderValue() string {
 	return ""
 }
 
+func (k EnvironmentID) Defined() bool {
+	return k != ""
+}
+
+func (k EnvironmentID) String() string {
+	return string(k)
+}
+
+func (k EnvironmentID) Compare(_ credential.AutoConfig) (credential.SDKCredential, credential.Status) {
+	// EnvironmentIDs should not change.
+	return nil, credential.Unchanged
+}
+
 // GetAuthorizationHeaderValue for AutoConfigKey returns the same string, since these keys are passed in
 // the Authorization header. Note that unlike the other kinds of authorization keys, this one is never
 // present in an incoming request; it is only used in requests from Relay to LaunchDarkly.
 func (k AutoConfigKey) GetAuthorizationHeaderValue() string {
 	return string(k)
 }
 
+func (k AutoConfigKey) Compare(_ credential.AutoConfig) (credential.SDKCredential, credential.Status) {
+	// AutoConfigKeys should not change.
+	return nil, credential.Unchanged
+}
+
+func (k AutoConfigKey) String() string {
+	return string(k)
+}
+
 // UnmarshalText allows the SDKKey type to be set from environment variables.
 func (k *SDKKey) UnmarshalText(data []byte) error {
 	*k = SDKKey(string(data))
@@ -87,6 +153,10 @@ func (k *AutoConfigKey) UnmarshalText(data []byte) error {
 	return nil
 }
 
+func (k AutoConfigKey) Defined() bool {
+	return k != ""
+}
+
 // OptLogLevel represents an optional log level parameter. It must match one of the level names "debug",
 // "info", "warn", or "error" (case-insensitive).
 //

config/config_from_env.go

@@ -46,7 +46,7 @@ func LoadConfigFromEnvironmentBase(c *Config, loggers ldlog.Loggers) ct.Validati
 	// The following properties have the same environment variable names in AutoConfigConfig and in
 	// OfflineModeConfig, because only one of those can be used at a time. We'll blank them out for
 	// whichever section is not being used.
-	if c.AutoConfig.Key != "" {
+	if c.AutoConfig.Key.Defined() {
 		c.OfflineMode.EnvAllowedOrigin = ct.OptStringList{}
 		c.OfflineMode.EnvAllowedHeader = ct.OptStringList{}
 		c.OfflineMode.EnvDatastorePrefix = ""
@@ -76,6 +76,19 @@ func LoadConfigFromEnvironmentBase(c *Config, loggers ldlog.Loggers) ct.Validati
 		c.Environment[envName] = &ec
 	}
 
+	for projKey := range reader.FindPrefixedValues("LD_FILTER_KEYS_") {
+		var fc FiltersConfig
+		if c.Filters[projKey] != nil {
+			fc = *c.Filters[projKey]
+		}
+		subReader := reader.WithVarNameSuffix(projKey)
+		subReader.ReadStruct(&fc, false)
+		if c.Filters == nil {
+			c.Filters = make(map[string]*FiltersConfig)
+		}
+		c.Filters[projKey] = &fc
+	}
+
 	useRedis := false
 	reader.Read("USE_REDIS", &useRedis)
 	if useRedis || c.Redis.Host != "" || c.Redis.URL.IsDefined() {

config/config_validation.go

@@ -21,6 +21,8 @@ var (
 	errRedisURLWithHostAndPort = errors.New("please specify Redis URL or host/port, but not both")
 	errRedisBadHostname        = errors.New("invalid Redis hostname")
 	errConsulTokenAndTokenFile = errors.New("Consul token must be specified as either an inline value or a file, but not both") //nolint:stylecheck
+	errAutoConfWithFilters     = errors.New("cannot configure filters if auto-configuration is enabled")
+	errMissingProjKey          = errors.New("when filters are configured, all environments must specify a 'projKey'")
 )
 
 func errEnvironmentWithNoSDKKey(envName string) error {
@@ -38,6 +40,18 @@ func errEnvWithoutDBDisambiguation(envName string, canUseTableName bool) error {
 	return fmt.Errorf("environment %q does not have a prefix specified for database storage", envName)
 }
 
+func errFilterUnknownProject(projKey string) error {
+	return fmt.Errorf("filters are configured for project '%s', but no environment references that project", projKey)
+}
+
+func errFilterEmptyKeys(projKey string) error {
+	return fmt.Errorf("filter key list for project '%s' cannot be empty", projKey)
+}
+
+func errFilterInvalidKey(projKey string, i int) error {
+	return fmt.Errorf("filter key [%d] for project '%s' is malformed (note: lists are comma-delimited)", i, projKey)
+}
+
 func warnEnvWithoutDBDisambiguation(envName string, canUseTableName bool) string {
 	return errEnvWithoutDBDisambiguation(envName, canUseTableName).Error() +
 		"; this would be an error if multiple environments were configured"
@@ -61,6 +75,7 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error {
 	validateConfigTLS(&result, c)
 	validateConfigEnvironments(&result, c)
 	validateConfigDatabases(&result, c, loggers)
+	validateConfigFilters(&result, c)
 
 	return result.GetError()
 }
@@ -126,6 +141,49 @@ func validateConfigEnvironments(result *ct.ValidationResult, c *Config) {
 	}
 }
 
+func validateConfigFilters(result *ct.ValidationResult, c *Config) {
+	if len(c.Filters) == 0 {
+		return
+	}
+	// If Auto Config is enabled, then filters will have no effect and should cause an error.
+	if c.AutoConfig.Key != "" {
+		result.AddError(nil, errAutoConfWithFilters)
+		return
+	}
+	for _, proj := range c.Environment {
+		if proj.ProjKey == "" {
+			result.AddError(nil, errMissingProjKey)
+			return
+		}
+	}
+	for projKey, conf := range c.Filters {
+		// For every project key defined by a [filter] section,
+		// that project key must be referenced by at least one environment.
+		foundProj := false
+		for _, e := range c.Environment {
+			if e.ProjKey == projKey {
+				foundProj = true
+				break
+			}
+		}
+		if !foundProj {
+			result.AddError(nil, errFilterUnknownProject(projKey))
+			continue
+		}
+
+		// The list of filter keys cannot be empty
+		if len(conf.Keys.Values()) == 0 {
+			result.AddError(nil, errFilterEmptyKeys(projKey))
+		} else {
+			// Filter keys cannot be empty strings
+			for i, k := range conf.Keys.Values() {
+				if k == "" {
+					result.AddError(nil, errFilterInvalidKey(projKey, i))
+				}
+			}
+		}
+	}
+}
 func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) {
 	normalizeRedisConfig(result, c)
 
@@ -172,7 +230,7 @@ func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldl
 			}
 		}
 
-	case c.AutoConfig.Key != "":
+	case c.AutoConfig.Key.Defined():
 		// Same as previous case, except that in auto-config mode we must assume that there are multiple environments.
 		if !strings.Contains(c.AutoConfig.EnvDatastorePrefix, AutoConfigEnvironmentIDPlaceholder) &&
 			!(c.DynamoDB.Enabled && strings.Contains(c.AutoConfig.EnvDatastoreTableName, AutoConfigEnvironmentIDPlaceholder)) {

docs/configuration.md

@@ -137,6 +137,7 @@ The Relay Proxy allows you to proxy any number of LaunchDarkly environments; the
 | `allowedHeader`  | `LD_ALLOWED_HEADER_MyEnvName` |  String  | If provided, adds the specify headers to the list of accepted headers for CORS requests. This variable can be provided multiple times per environment (if using the `LD_ALLOWED_HEADER_MyEnvName` variable, specify a comma-delimited list). |
 | `logLevel`       | `LD_LOG_LEVEL_MyEnvName`      |  String  | Should be `debug`, `info`, `warn`, `error`, or `none`. **See: [Logging](./logging.md)**                                                                                                                                                      |
 | `ttl`            | `LD_TTL_MyEnvName`            | Duration | HTTP caching TTL for the PHP polling endpoints. **See: [Using PHP](./php.md)**                                                                                                                                                               |
+| `projKey`        | `LD_PROJ_KEY_MyEnvName`       |  String  | Project key for this environment. Required if any filters are defined. Filtering is an Enterprise-only feature.                                                                                                                              |
 
 In the following examples, there are two environments, each of which has a server-side SDK key and a mobile key. Debug-level logging is enabled for the second one.
 
@@ -162,6 +163,16 @@ LD_ENV_Spree_Project_Test=SPREE_TEST_SDK_KEY
 LD_MOBILE_KEY_Spree_Project_Test=SPREE_TEST_MOBILE_KEY
 ```
 
+### File section: `[Filters "PROJECT-KEY"]`
+
+To learn more, read [Filters](TBD).
+
+
+| Property in file | Environment var            |  Type   | Default | Description                                                                                                                                                                                                                         |
+|------------------|----------------------------|:-------:|:--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `keys`           | `LD_FILTER_KEYS_MyProjKey` | String  |         | Specify one or more filter keys for this project _(1)_. This variable can be provided multiple times, or specified using a comma-delimited list (if using the `LD_FILTER_KEYS_MyProjKey` variable, specify a comma-delimited list.) |
+
+_(1)_ SDKs may request filtered environments identified by any of these keys, as well as the default unfiltered environment.
 
 ### File section: `[Redis]`
 

go.mod

@@ -26,7 +26,6 @@ require (
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/kardianos/minwinsvc v1.0.2
-	github.com/launchdarkly/api-client-go/v12 v12.0.0
 	github.com/launchdarkly/eventsource v1.7.1
 	github.com/launchdarkly/go-configtypes v1.1.0
 	github.com/launchdarkly/go-jsonstream/v3 v3.0.0
@@ -36,14 +35,14 @@ require (
 	github.com/launchdarkly/go-server-sdk-dynamodb/v3 v3.0.2
 	github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2
 	github.com/launchdarkly/go-server-sdk-redis-redigo/v2 v2.0.1
-	github.com/launchdarkly/go-server-sdk/v6 v6.0.3
+	github.com/launchdarkly/go-server-sdk/v6 v6.1.0-alpha.pub.3
 	github.com/launchdarkly/go-test-helpers/v3 v3.0.2
 	github.com/launchdarkly/opencensus-go-exporter-stackdriver v0.14.2
 	github.com/pborman/uuid v1.2.1
 	github.com/prometheus/client_golang v1.14.0 // indirect; override to address CVE-2022-21698
 	github.com/stretchr/testify v1.8.2
 	go.opencensus.io v0.24.0
-	golang.org/x/net v0.8.0 // indirect; override to address CVE-2022-41723
+	golang.org/x/net v0.9.0 // indirect; override to address CVE-2022-41723
 	golang.org/x/sync v0.1.0
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/launchdarkly/go-server-sdk.v5 v5.10.1
@@ -61,10 +60,13 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect
 	github.com/aws/smithy-go v1.13.5 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
 )
 
-require github.com/goreleaser/goreleaser v1.15.2
+require (
+	github.com/goreleaser/goreleaser v1.15.2
+	github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e
+)
 
 require (
 	cloud.google.com/go/compute v1.19.0 // indirect
@@ -236,9 +238,9 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
-	github.com/prometheus/statsd_exporter v0.23.1 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/rivo/uniseg v0.4.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sasha-s/go-csync v0.0.0-20210812194225-61421b77c44b // indirect
@@ -257,7 +259,7 @@ require (
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
 	github.com/theupdateframework/go-tuf v0.5.2-0.20220930112810-3890c1e7ace4 // indirect
-	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
@@ -268,14 +270,14 @@ require (
 	gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect
 	go.mongodb.org/mongo-driver v1.10.2 // indirect
 	gocloud.dev v0.28.0 // indirect
-	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/mod v0.9.0 // indirect
-	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/term v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.114.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect