Skip to content

Commit

Permalink
feat: Add support for sending/receiving gzip event payloads (#364)
Browse files Browse the repository at this point in the history
  • Loading branch information
@keelerm84
keelerm84 authored Jul 25, 2024
1 parent 3e16771 commit 4803e76
Show file tree
Hide file tree
Showing 21 changed files with 447 additions and 83 deletions.
11 changes: 6 additions & 5 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,11 +184,12 @@ type OfflineModeConfig struct {
// variables, individual fields are not documented here; instead, see the `README.md` section on
// configuration.
type EventsConfig struct {
EventsURI ct.OptURLAbsolute `conf:"EVENTS_HOST"`
SendEvents bool `conf:"USE_EVENTS"`
FlushInterval ct.OptDuration `conf:"EVENTS_FLUSH_INTERVAL"`
Capacity ct.OptIntGreaterThanZero `conf:"EVENTS_CAPACITY"`
InlineUsers bool `conf:"EVENTS_INLINE_USERS"`
EventsURI ct.OptURLAbsolute `conf:"EVENTS_HOST"`
SendEvents bool `conf:"USE_EVENTS"`
FlushInterval ct.OptDuration `conf:"EVENTS_FLUSH_INTERVAL"`
Capacity ct.OptIntGreaterThanZero `conf:"EVENTS_CAPACITY"`
InlineUsers bool `conf:"EVENTS_INLINE_USERS"`
MaxInboundPayloadSize ct.OptBase2Bytes `conf:"EVENTS_MAX_INBOUND_PAYLOAD_SIZE"`
}

// RedisConfig configures the optional Redis integration.
Expand Down
12 changes: 12 additions & 0 deletions config/config_validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ var (
errFileDataWithAutoConf = errors.New("cannot specify both auto-configuration key and file data source")
errOfflineModePropertiesWithNoFile = errors.New("must specify offline mode filename if other offline mode properties are set")
errOfflineModeWithEnvironments = errors.New("cannot configure specific environments if offline mode is enabled")
errMaxInboundPayloadSize = errors.New("max inbound payload size must be greater than zero")
errAutoConfWithoutDBDisambig = errors.New(`when using auto-configuration with database storage, database prefix (or,` +
` if using DynamoDB, table name) must be specified and must contain "` + AutoConfigEnvironmentIDPlaceholder + `"`)
errRedisURLWithHostAndPort = errors.New("please specify Redis URL or host/port, but not both")
Expand Down Expand Up @@ -80,6 +81,7 @@ func ValidateConfig(c *Config, loggers ldlog.Loggers) error {
validateConfigFilters(&result, c)
validateOfflineMode(&result, c)
validateCredentialCleanupInterval(&result, c)
validateMaxInboundPayloadSize(&result, c)

return result.GetError()
}
Expand Down Expand Up @@ -206,6 +208,16 @@ func validateCredentialCleanupInterval(result *ct.ValidationResult, c *Config) {
}
}
}

func validateMaxInboundPayloadSize(result *ct.ValidationResult, c *Config) {
if c.Events.MaxInboundPayloadSize.IsDefined() {
size := c.Events.MaxInboundPayloadSize.GetOrElse(0)
if size <= 0 {
result.AddError(nil, errMaxInboundPayloadSize)
}
}
}

func validateConfigDatabases(result *ct.ValidationResult, c *Config, loggers ldlog.Loggers) {
normalizeRedisConfig(result, c)

Expand Down
33 changes: 28 additions & 5 deletions config/test_data_configs_valid_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ func makeValidConfigs() []testDataValidConfig {
makeValidConfigExplicitOldDefaultBaseURI(),
makeValidConfigAutoConfig(),
makeValidConfigAutoConfigWithDatabase(),
makeValidConfigMaxInboundPayloadSize("50KiB"),
makeValidConfigMaxInboundPayloadSize("7MiB"),
makeValidConfigMaxInboundPayloadSize("10GiB"),
makeValidConfigOfflineModeMinimal(),
makeValidConfigOfflineModeWithMonitoringInterval("100ms"),
makeValidConfigOfflineModeWithMonitoringInterval("1s"),
Expand Down Expand Up @@ -118,11 +121,12 @@ func makeValidConfigAllBaseProperties() testDataValidConfig {
ExpiredCredentialCleanupInterval: ct.NewOptDuration(1 * time.Minute),
}
c.Events = EventsConfig{
SendEvents: true,
EventsURI: newOptURLAbsoluteMustBeValid("http://events"),
FlushInterval: ct.NewOptDuration(120 * time.Second),
Capacity: mustOptIntGreaterThanZero(500),
InlineUsers: true,
SendEvents: true,
EventsURI: newOptURLAbsoluteMustBeValid("http://events"),
FlushInterval: ct.NewOptDuration(120 * time.Second),
Capacity: mustOptIntGreaterThanZero(500),
InlineUsers: true,
MaxInboundPayloadSize: ct.OptBase2Bytes{},
}
c.Environment = map[string]*EnvConfig{
"earth": {
Expand Down Expand Up @@ -356,6 +360,25 @@ FileDataSource = my-file-path
return c
}

func makeValidConfigMaxInboundPayloadSize(size string) testDataValidConfig {
bytes, err := ct.NewOptBase2BytesFromString(size)
if err != nil {
panic(err)
}
c := testDataValidConfig{name: "file data properties"}
c.makeConfig = func(c *Config) {
c.Events.MaxInboundPayloadSize = bytes
}
c.envVars = map[string]string{
"EVENTS_MAX_INBOUND_PAYLOAD_SIZE": size,
}
c.fileContent = `
[Events]
MaxInboundPayloadSize = ` + size

return c
}

func MustOptDurationFromString(duration string) ct.OptDuration {
opt, err := ct.NewOptDurationFromString(duration)
if err != nil {
Expand Down
16 changes: 9 additions & 7 deletions docs/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,17 @@ Note that the last three properties have the same meanings and the same environm

To learn more, read [Forwarding events](./events.md).

| Property in file | Environment var | Type | Default | Description |
|------------------|-------------------------|:--------:|:--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `sendEvents` | `USE_EVENTS` | Boolean | `false` | When enabled, the Relay Proxy will send analytic events it receives to LaunchDarkly, unless offline mode is enabled. |
| `eventsUri` | `EVENTS_HOST` | URI | _(7)_ | URI for the LaunchDarkly events service |
| `flushInterval` | `EVENTS_FLUSH_INTERVAL` | Duration | `5s` | Controls how long the SDK buffers events before sending them back to our server. If your server generates many events per second, we suggest decreasing the flush interval and/or increasing capacity to meet your needs. |
| `capacity` | `EVENTS_CAPACITY` | Number | `1000` | Maximum number of events to accumulate for each flush interval. |
| `inlineUsers` | `EVENTS_INLINE_USERS` | Boolean | `false` | When enabled, individual events (if full event tracking is enabled for the feature flag) will contain all non-private user attributes. |
| Property in file | Environment var | Type | Default | Description |
|-------------------------------|--------------------------------------|:--------:|:--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `sendEvents` | `USE_EVENTS` | Boolean | `false` | When enabled, the Relay Proxy will send analytic events it receives to LaunchDarkly, unless offline mode is enabled. |
| `eventsUri` | `EVENTS_HOST` | URI | _(7)_ | URI for the LaunchDarkly events service |
| `flushInterval` | `EVENTS_FLUSH_INTERVAL` | Duration | `5s` | Controls how long the SDK buffers events before sending them back to our server. If your server generates many events per second, we suggest decreasing the flush interval and/or increasing capacity to meet your needs. |
| `capacity` | `EVENTS_CAPACITY` | Number | `1000` | Maximum number of events to accumulate for each flush interval. |
| `inlineUsers` | `EVENTS_INLINE_USERS` | Boolean | `false` | When enabled, individual events (if full event tracking is enabled for the feature flag) will contain all non-private user attributes. |
| `maxInboundPayloadSize` | `EVENTS_MAX_INBOUND_PAYLOAD_SIZE` | Unit | _(8)_ | Maximum size of an event payload the Relay Proxy will accept from an SDK. |

_(7)_ See note _(1)_ above. The default value for `eventsUri` is `https://events.launchdarkly.com`.
_(8)_ The `maxInboundPayloadSize` setting is used to limit the size of the payload that the Relay Proxy will accept from an SDK. This is an optional safety feature to prevent the Relay Proxy from being overwhelmed by a very large payload. The default value is `0B` which provides no restriction on the payload size. The value should be a number followed by a unit: `B` for bytes, `KiB` for kibibytes, `MiB` for mebibytes, `GiB` for gibibytes, `TiB` for tebibytes, `PiB` for pebibytes, or `EiB` for exbibytes. For example, `100MiB` is 100 mebibytes.


### File section: `[Environment "NAME"]`
Expand Down
17 changes: 9 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/launchdarkly/ld-relay/v8

go 1.20
go 1.21

require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2
Expand All @@ -12,7 +12,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0
github.com/cyphar/filepath-securejoin v0.2.4
github.com/fatih/color v1.15.0 // indirect
github.com/fsnotify/fsnotify v1.7.0
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-redis/redis/v8 v8.11.5
github.com/gomodule/redigo v1.8.9
github.com/google/uuid v1.5.0 // indirect
Expand All @@ -25,15 +25,15 @@ require (
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/kardianos/minwinsvc v1.0.2
github.com/launchdarkly/eventsource v1.7.1
github.com/launchdarkly/go-configtypes v1.1.0
github.com/launchdarkly/go-configtypes v1.2.0
github.com/launchdarkly/go-jsonstream/v3 v3.0.0
github.com/launchdarkly/go-sdk-common/v3 v3.1.0
github.com/launchdarkly/go-sdk-events/v3 v3.2.0
github.com/launchdarkly/go-sdk-events/v3 v3.4.0
github.com/launchdarkly/go-server-sdk-consul/v3 v3.0.0
github.com/launchdarkly/go-server-sdk-dynamodb/v4 v4.0.0
github.com/launchdarkly/go-server-sdk-evaluation/v3 v3.0.0
github.com/launchdarkly/go-server-sdk-redis-redigo/v3 v3.0.0
github.com/launchdarkly/go-server-sdk/v7 v7.1.0
github.com/launchdarkly/go-server-sdk/v7 v7.6.0
github.com/launchdarkly/go-test-helpers/v3 v3.0.2
github.com/launchdarkly/opencensus-go-exporter-stackdriver v0.14.2
github.com/pborman/uuid v1.2.1
Expand All @@ -59,7 +59,10 @@ require (
golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb
)

require github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e
require (
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
github.com/launchdarkly/api-client-go/v13 v13.0.1-0.20230420175109-f5469391a13e
)

require (
cloud.google.com/go/compute v1.23.3 // indirect
Expand Down Expand Up @@ -123,8 +126,6 @@ require (
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/DataDog/dd-trace-go.v1 v1.56.1 // indirect
gopkg.in/launchdarkly/go-jsonstream.v1 v1.0.1 // indirect
gopkg.in/launchdarkly/go-sdk-common.v2 v2.5.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading

0 comments on commit 4803e76

Please sign in to comment.