feat(autoedit): add discarded events (#6564)

Added `discarded` analytics events for the autoedits provider that would allow us to know how often and why autoedit suggestions are not shown to users after they have been fetched from the backend.
sourcegraph · Jan 9, 2025 · 6cbe896 · 6cbe896
1 parent 825ff80
commit 6cbe896
Show file tree

Hide file tree

Showing 3 changed files with 106 additions and 6 deletions.
diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts
@@ -20,6 +20,7 @@ import { getDecorationInfo } from '../renderer/diff-utils'
 import {
     AutoeditAnalyticsLogger,
     type AutoeditRequestID,
+    autoeditDiscardReason,
     autoeditSource,
     autoeditTriggerKind,
 } from './analytics-logger'
@@ -301,10 +302,40 @@ describe('AutoeditAnalyticsLogger', () => {
     it('logs `discarded` if the suggestion was not suggested for any reason', () => {
         const requestId = autoeditLogger.createRequest(getRequestStartMetadata())
         autoeditLogger.markAsContextLoaded({ requestId, payload: { contextSummary: undefined } })
-        autoeditLogger.markAsDiscarded(requestId)
+        autoeditLogger.markAsDiscarded({
+            requestId,
+            discardReason: autoeditDiscardReason.emptyPrediction,
+        })
 
         expect(recordSpy).toHaveBeenCalledTimes(1)
         expect(recordSpy).toHaveBeenCalledWith('cody.autoedit', 'discarded', expect.any(Object))
+
+        const discardedEventPayload = recordSpy.mock.calls[0].at(2)
+        expect(discardedEventPayload).toMatchInlineSnapshot(`
+          {
+            "billingMetadata": {
+              "category": "core",
+              "product": "cody",
+            },
+            "interactionID": undefined,
+            "metadata": {
+              "discardReason": 2,
+              "otherCompletionProviderEnabled": 0,
+              "recordsPrivateMetadataTranscript": 0,
+              "triggerKind": 1,
+            },
+            "privateMetadata": {
+              "codeToRewrite": "Code to rewrite",
+              "contextSummary": undefined,
+              "gatewayLatency": undefined,
+              "languageId": "typescript",
+              "model": "autoedit-model",
+              "otherCompletionProviders": [],
+              "upstreamLatency": undefined,
+            },
+            "version": 0,
+          }
+        `)
     })
 
     it('handles invalid transitions by logging debug events (invalidTransitionToXYZ)', () => {

diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.ts
@@ -165,6 +165,19 @@ export const autoeditSource = {
 /** We use numeric keys to send these to the analytics backend */
 type AutoeditSourceMetadata = (typeof autoeditSource)[keyof typeof autoeditSource]
 
+export const autoeditDiscardReason = {
+    clientAborted: 1,
+    emptyPrediction: 2,
+    predictionEqualsCodeToRewrite: 3,
+    recentEdits: 4,
+    suffixOverlap: 5,
+    emptyPredictionAfterInlineCompletionExtraction: 6,
+    noActiveEditor: 7,
+} as const
+
+/** We use numeric keys to send these to the analytics backend */
+type AutoeditDiscardReasonMetadata = (typeof autoeditDiscardReason)[keyof typeof autoeditDiscardReason]
+
 interface AutoeditLoadedMetadata extends AutoeditContextLoadedMetadata {
     /**
      * An ID to uniquely identify a suggest autoedit. Note: It is possible for this ID to be part
@@ -229,7 +242,9 @@ interface AutoeditAcceptedEventPayload
         Omit<CodeGenEventMetadata, 'charsInserted' | 'charsDeleted'> {}
 
 interface AutoeditRejectedEventPayload extends AutoEditFinalMetadata {}
-interface AutoeditDiscardedEventPayload extends AutoeditContextLoadedMetadata {}
+interface AutoeditDiscardedEventPayload extends AutoeditContextLoadedMetadata {
+    discardReason: AutoeditDiscardReasonMetadata
+}
 
 /**
  * An ephemeral ID for a single “request” from creation to acceptance or rejection.
@@ -595,8 +610,22 @@ export class AutoeditAnalyticsLogger {
         }
     }
 
-    public markAsDiscarded(requestId: AutoeditRequestID): void {
-        const result = this.tryTransitionTo(requestId, 'discarded', currentRequest => currentRequest)
+    public markAsDiscarded({
+        requestId,
+        discardReason,
+    }: {
+        requestId: AutoeditRequestID
+        discardReason: AutoeditDiscardReasonMetadata
+    }): void {
+        const result = this.tryTransitionTo(requestId, 'discarded', request => {
+            return {
+                ...request,
+                payload: {
+                    ...request.payload,
+                    discardReason: discardReason,
+                },
+            }
+        })
 
         if (result?.updatedRequest) {
             this.writeAutoeditRequestEvent('discarded', result.updatedRequest)

diff --git a/vscode/src/autoedits/autoedits-provider.ts b/vscode/src/autoedits/autoedits-provider.ts
@@ -21,6 +21,7 @@ import { createAutoeditsModelAdapter } from './adapters/create-adapter'
 import {
     type AutoeditRequestID,
     autoeditAnalyticsLogger,
+    autoeditDiscardReason,
     autoeditSource,
     autoeditTriggerKind,
     getTimeNowInMillis,
@@ -254,11 +255,30 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
             prompt,
             codeToReplaceData,
         })
-        if (abortSignal?.aborted || !initialPrediction) {
+
+        if (abortSignal?.aborted) {
             autoeditsOutputChannelLogger.logDebugIfVerbose(
                 'provideInlineCompletionItems',
-                'aborted after getPrediction'
+                'client aborted after getPrediction'
             )
+
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.clientAborted,
+            })
+            return null
+        }
+
+        if (initialPrediction === undefined || initialPrediction.length === 0) {
+            autoeditsOutputChannelLogger.logDebugIfVerbose(
+                'provideInlineCompletionItems',
+                'received empty prediction'
+            )
+
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.emptyPrediction,
+            })
             return null
         }
 
@@ -288,6 +308,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
                 'provideInlineCompletionItems',
                 'prediction equals to code to rewrite'
             )
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.predictionEqualsCodeToRewrite,
+            })
             return null
         }
 
@@ -302,6 +326,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
                 'provideInlineCompletionItems',
                 'based on recent edits'
             )
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.recentEdits,
+            })
             return null
         }
 
@@ -318,6 +346,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
                 'provideInlineCompletionItems',
                 'skip because the prediction equals to code to rewrite'
             )
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.suffixOverlap,
+            })
             return null
         }
 
@@ -337,6 +369,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
                 'provideInlineCompletionItems',
                 'no suggestion to render'
             )
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.emptyPredictionAfterInlineCompletionExtraction,
+            })
             return null
         }
 
@@ -346,6 +382,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
                 'provideInlineCompletionItems',
                 'no active editor'
             )
+            autoeditAnalyticsLogger.markAsDiscarded({
+                requestId,
+                discardReason: autoeditDiscardReason.noActiveEditor,
+            })
             return null
         }