From 9d41848d039385a12ae88d298cd6a8cdbc72f0ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Sinnbeck?= Date: Fri, 16 Feb 2024 13:37:32 +0100 Subject: [PATCH 1/6] remove extra closure --- src/Pulse.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Pulse.php b/src/Pulse.php index 6abe27e6..a3e14fee 100644 --- a/src/Pulse.php +++ b/src/Pulse.php @@ -311,7 +311,7 @@ public function ingest(): int $odds = $this->app->make('config')->get('pulse.ingest.trim.lottery') ?? $this->app->make('config')->get('pulse.ingest.trim_lottery'); Lottery::odds(...$odds) - ->winner(fn () => $this->rescue(fn () => $ingest->trim(...))) + ->winner(fn () => $this->rescue($ingest->trim(...))) ->choose(); $this->flush(); From e82e26784b740c7b053cd5023c7e4d0cd4094655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Sinnbeck?= Date: Fri, 16 Feb 2024 14:24:30 +0100 Subject: [PATCH 2/6] disable lottery for test --- tests/Feature/Storage/DatabaseStorageTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Feature/Storage/DatabaseStorageTest.php b/tests/Feature/Storage/DatabaseStorageTest.php index b04d04dc..b2f0b443 100644 --- a/tests/Feature/Storage/DatabaseStorageTest.php +++ b/tests/Feature/Storage/DatabaseStorageTest.php @@ -464,6 +464,7 @@ }); it('collapses values with the same key into a single upsert', function () { + Config::set('pulse.ingest.trim.lottery', [0, 1]); $bindings = []; DB::listen(function (QueryExecuted $event) use (&$bindings) { $bindings = $event->bindings; @@ -480,4 +481,4 @@ $values = Pulse::ignore(fn () => DB::table('pulse_values')->get()); expect($values)->toHaveCount(1); expect($values[0]->value)->toBe('345'); -}); +})->only(); From b21253102c180e51adc921d61aa2c0672cc53e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Sinnbeck?= Date: Fri, 16 Feb 2024 14:25:40 +0100 Subject: [PATCH 3/6] remove only call --- tests/Feature/Storage/DatabaseStorageTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Feature/Storage/DatabaseStorageTest.php b/tests/Feature/Storage/DatabaseStorageTest.php index b2f0b443..aef9aded 100644 --- a/tests/Feature/Storage/DatabaseStorageTest.php +++ b/tests/Feature/Storage/DatabaseStorageTest.php @@ -481,4 +481,4 @@ $values = Pulse::ignore(fn () => DB::table('pulse_values')->get()); expect($values)->toHaveCount(1); expect($values[0]->value)->toBe('345'); -})->only(); +}); From 56af51e503663cf44a671857d61684cff14914c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Sinnbeck?= Date: Fri, 16 Feb 2024 14:52:27 +0100 Subject: [PATCH 4/6] add test to ensure trimming is called --- tests/Feature/PulseTest.php | 11 +++++++++++ tests/StorageFake.php | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/Feature/PulseTest.php b/tests/Feature/PulseTest.php index a233fb59..7821c46d 100644 --- a/tests/Feature/PulseTest.php +++ b/tests/Feature/PulseTest.php @@ -34,6 +34,17 @@ expect($storage->stored[1]->key)->toBe('keep'); }); +it('can trim records', function () { + App::instance(Storage::class, $storage = new StorageFake); + + Pulse::record('foo', 'delete', 0, now()->subMonth()); + Pulse::record('foo', 'keep', 0); + + Pulse::ingest(); + + expect($storage->stored)->toHaveCount(1); +}); + it('can lazily capture entries', function () { App::instance(Storage::class, $storage = new StorageFake); diff --git a/tests/StorageFake.php b/tests/StorageFake.php index c9c8684a..7ace2561 100644 --- a/tests/StorageFake.php +++ b/tests/StorageFake.php @@ -31,7 +31,7 @@ public function store(Collection $items): void */ public function trim(): void { - // + $this->stored = $this->stored->reject(fn($record) => $record->timestamp < now()->subWeek()->timestamp); } /** From 1362cb633f64864051844d0407139362611dcac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Sinnbeck?= Date: Fri, 16 Feb 2024 14:56:00 +0100 Subject: [PATCH 5/6] change comparison to match with database query --- tests/StorageFake.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/StorageFake.php b/tests/StorageFake.php index 7ace2561..5b68df12 100644 --- a/tests/StorageFake.php +++ b/tests/StorageFake.php @@ -31,7 +31,7 @@ public function store(Collection $items): void */ public function trim(): void { - $this->stored = $this->stored->reject(fn($record) => $record->timestamp < now()->subWeek()->timestamp); + $this->stored = $this->stored->reject(fn($record) => $record->timestamp <= now()->subWeek()->timestamp); } /** From 8e67451d25f00d90e534319231c13772bc2185be Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Mon, 19 Feb 2024 15:14:40 +1100 Subject: [PATCH 6/6] Only collect queries we care about to remove trimming concerns --- tests/Feature/Storage/DatabaseStorageTest.php | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/Feature/Storage/DatabaseStorageTest.php b/tests/Feature/Storage/DatabaseStorageTest.php index aef9aded..f235a199 100644 --- a/tests/Feature/Storage/DatabaseStorageTest.php +++ b/tests/Feature/Storage/DatabaseStorageTest.php @@ -123,9 +123,12 @@ }); it('combines duplicate count aggregates before upserting', function () { - Config::set('pulse.ingest.trim.lottery', [0, 1]); $queries = collect(); - DB::listen(fn ($query) => $queries[] = $query); + DB::listen(function (QueryExecuted $event) use (&$queries) { + if (str_starts_with($event->sql, 'insert')) { + $queries[] = $event; + } + }); Pulse::record('type', 'key1')->count(); Pulse::record('type', 'key1')->count(); @@ -150,9 +153,12 @@ }); it('combines duplicate min aggregates before upserting', function () { - Config::set('pulse.ingest.trim.lottery', [0, 1]); $queries = collect(); - DB::listen(fn ($query) => $queries[] = $query); + DB::listen(function (QueryExecuted $event) use (&$queries) { + if (str_starts_with($event->sql, 'insert')) { + $queries[] = $event; + } + }); Pulse::record('type', 'key1', 200)->min(); Pulse::record('type', 'key1', 100)->min(); @@ -177,9 +183,12 @@ }); it('combines duplicate max aggregates before upserting', function () { - Config::set('pulse.ingest.trim.lottery', [0, 1]); $queries = collect(); - DB::listen(fn ($query) => $queries[] = $query); + DB::listen(function (QueryExecuted $event) use (&$queries) { + if (str_starts_with($event->sql, 'insert')) { + $queries[] = $event; + } + }); Pulse::record('type', 'key1', 100)->max(); Pulse::record('type', 'key1', 300)->max(); @@ -204,9 +213,12 @@ }); it('combines duplicate sum aggregates before upserting', function () { - Config::set('pulse.ingest.trim.lottery', [0, 1]); $queries = collect(); - DB::listen(fn ($query) => $queries[] = $query); + DB::listen(function (QueryExecuted $event) use (&$queries) { + if (str_starts_with($event->sql, 'insert')) { + $queries[] = $event; + } + }); Pulse::record('type', 'key1', 100)->sum(); Pulse::record('type', 'key1', 300)->sum(); @@ -231,9 +243,12 @@ }); it('combines duplicate average aggregates before upserting', function () { - Config::set('pulse.ingest.trim.lottery', [0, 1]); $queries = collect(); - DB::listen(fn ($query) => $queries[] = $query); + DB::listen(function (QueryExecuted $event) use (&$queries) { + if (str_starts_with($event->sql, 'insert')) { + $queries[] = $event; + } + }); Pulse::record('type', 'key1', 100)->avg(); Pulse::record('type', 'key1', 300)->avg(); @@ -464,10 +479,11 @@ }); it('collapses values with the same key into a single upsert', function () { - Config::set('pulse.ingest.trim.lottery', [0, 1]); $bindings = []; DB::listen(function (QueryExecuted $event) use (&$bindings) { - $bindings = $event->bindings; + if (str_starts_with($event->sql, 'insert')) { + $bindings = $event->bindings; + } }); Pulse::set('read_counter', 'post:321', 123);