ingest/cdp: Update the default config for BufferedStorageBackend base…

…d on the benchmarking results (#5572)
stellar · Jan 15, 2025 · bd7c1ad · bd7c1ad
1 parent c29e294
commit bd7c1ad
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 20 deletions.
diff --git a/ingest/cdp/producer.go b/ingest/cdp/producer.go
@@ -34,13 +34,9 @@ func DefaultBufferedStorageBackendConfig(ledgersPerFile uint32) ledgerbackend.Bu
 	}
 
 	switch {
-	case ledgersPerFile < 2:
-		config.BufferSize = 500
-		config.NumWorkers = 5
-		return config
-	case ledgersPerFile < 101:
-		config.BufferSize = 10
-		config.NumWorkers = 5
+	case ledgersPerFile < 64:
+		config.BufferSize = 100
+		config.NumWorkers = 10
 		return config
 	default:
 		config.BufferSize = 10

diff --git a/ingest/cdp/producer_test.go b/ingest/cdp/producer_test.go
@@ -24,15 +24,8 @@ func TestDefaultBSBConfigs(t *testing.T) {
 	smallConfig := ledgerbackend.BufferedStorageBackendConfig{
 		RetryLimit: 5,
 		RetryWait:  30 * time.Second,
-		BufferSize: 500,
-		NumWorkers: 5,
-	}
-
-	mediumConfig := ledgerbackend.BufferedStorageBackendConfig{
-		RetryLimit: 5,
-		RetryWait:  30 * time.Second,
-		BufferSize: 10,
-		NumWorkers: 5,
+		BufferSize: 100,
+		NumWorkers: 10,
 	}
 
 	largeConfig := ledgerbackend.BufferedStorageBackendConfig{
@@ -43,10 +36,8 @@ func TestDefaultBSBConfigs(t *testing.T) {
 	}
 
 	assert.Equal(t, DefaultBufferedStorageBackendConfig(1), smallConfig)
-	assert.Equal(t, DefaultBufferedStorageBackendConfig(2), mediumConfig)
-	assert.Equal(t, DefaultBufferedStorageBackendConfig(100), mediumConfig)
-	assert.Equal(t, DefaultBufferedStorageBackendConfig(101), largeConfig)
-	assert.Equal(t, DefaultBufferedStorageBackendConfig(1000), largeConfig)
+	assert.Equal(t, DefaultBufferedStorageBackendConfig(64), largeConfig)
+	assert.Equal(t, DefaultBufferedStorageBackendConfig(512), largeConfig)
 }
 
 func TestBSBProducerFn(t *testing.T) {