Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Retrying logic for context-eval #6499

Closed
wants to merge 1 commit into from
Closed

Retrying logic for context-eval #6499

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 160 additions & 65 deletions agent/src/cli/command-bench/strategy-chat-context.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,79 +83,174 @@
})
}

// async function runContextCommand(
// clientOpts: ClientOptions,
// examples: Example[]
// ): Promise<ExampleOutput[]> {
// const completionsClient = new SourcegraphNodeCompletionsClient()
// const exampleOutputs: ExampleOutput[] = []

// for (const example of examples) {
// const { targetRepoRevs, query: origQuery } = example
// const repoNames = targetRepoRevs.map(repoRev => repoRev.repoName)
// const repoIDNames = await graphqlClient.getRepoIds(repoNames, repoNames.length + 10)
// if (isError(repoIDNames)) {
// throw new Error(`getRepoIds failed for [${repoNames.join(',')}]: ${repoIDNames}`)
// }
// if (repoIDNames.length !== repoNames.length) {
// throw new Error(
// `repoIDs.length (${repoIDNames.length}) !== repoNames.length (${
// repoNames.length
// }), repoNames were (${repoNames.join(', ')})`
// )
// }
// const repoIDs = repoIDNames.map(repoIDName => repoIDName.id)

// let query = origQuery
// if (clientOpts.rewrite) {
// query = await rewriteKeywordQuery(
// completionsClient,
// PromptString.unsafe_fromUserQuery(origQuery)
// )
// }

// const resultsResp = await graphqlClient.contextSearchEvalDebug({
// repoIDs,
// query,
// filePatterns: [],
// codeResultsCount: clientOpts.codeResultsCount,
// textResultsCount: clientOpts.textResultsCount,
// })

// if (isError(resultsResp)) {
// throw new Error(
// `contextSearch failed for repos [${repoNames.join(
// ','
// )}] and query "${query}": ${resultsResp}`
// )
// }
// if (resultsResp === null) {
// throw new Error(
// `contextSearch failed for repos [${repoNames.join(
// ','
// )}] and query "${query}": null results`
// )
// }

// const results = resultsResp ?? []
// const actualContext: EvalContextItem[] = []
// for (const contextList of results) {
// actualContext.push(
// ...contextList.contextList.map(result => ({
// repoName: result.repoName.replace(/^github\.com\//, ''),
// path: result.path,
// startLine: result.startLine,
// endLine: result.endLine,
// content: result.content,
// retriever: contextList.name,
// }))
// )
// }
// exampleOutputs.push({
// ...example,
// actualContext,
// })
// }

// return exampleOutputs
// }

async function sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}

async function runContextCommand(
clientOpts: ClientOptions,
examples: Example[]
): Promise<ExampleOutput[]> {
const completionsClient = new SourcegraphNodeCompletionsClient()
const exampleOutputs: ExampleOutput[] = []
const completionsClient = new SourcegraphNodeCompletionsClient();
const exampleOutputs: ExampleOutput[] = [];

for (const example of examples) {
const { targetRepoRevs, query: origQuery } = example
const repoNames = targetRepoRevs.map(repoRev => repoRev.repoName)
const repoIDNames = await graphqlClient.getRepoIds(repoNames, repoNames.length + 10)
if (isError(repoIDNames)) {
throw new Error(`getRepoIds failed for [${repoNames.join(',')}]: ${repoIDNames}`)
}
if (repoIDNames.length !== repoNames.length) {
throw new Error(
`repoIDs.length (${repoIDNames.length}) !== repoNames.length (${
repoNames.length
}), repoNames were (${repoNames.join(', ')})`
)
}
const repoIDs = repoIDNames.map(repoIDName => repoIDName.id)

let query = origQuery
if (clientOpts.rewrite) {
query = await rewriteKeywordQuery(
completionsClient,
PromptString.unsafe_fromUserQuery(origQuery)
)
}
const { targetRepoRevs, query: origQuery } = example;
const repoNames = targetRepoRevs.map(repoRev => repoRev.repoName);
let success = false;
let attempt = 0;

const resultsResp = await graphqlClient.contextSearchEvalDebug({
repoIDs,
query,
filePatterns: [],
codeResultsCount: clientOpts.codeResultsCount,
textResultsCount: clientOpts.textResultsCount,
})

if (isError(resultsResp)) {
throw new Error(
`contextSearch failed for repos [${repoNames.join(
','
)}] and query "${query}": ${resultsResp}`
)
}
if (resultsResp === null) {
throw new Error(
`contextSearch failed for repos [${repoNames.join(
','
)}] and query "${query}": null results`
)
while (!success && attempt < 3) { // Retry up to 3 times
try {
const repoIDNames = await graphqlClient.getRepoIds(repoNames, repoNames.length + 10);
if (isError(repoIDNames)) {
throw new Error(`getRepoIds failed for [${repoNames.join(',')}]: ${repoIDNames}`);
}
if (repoIDNames.length !== repoNames.length) {
throw new Error(
`repoIDs.length (${repoIDNames.length}) !== repoNames.length (${repoNames.length}), repoNames were (${repoNames.join(', ')})`
);
}
const repoIDs = repoIDNames.map(repoIDName => repoIDName.id);

let query = origQuery;
if (clientOpts.rewrite) {
query = await rewriteKeywordQuery(
completionsClient,
PromptString.unsafe_fromUserQuery(origQuery)

Check failure on line 197 in agent/src/cli/command-bench/strategy-chat-context.ts

View workflow job for this annotation

GitHub Actions / safe-prompts-lint 

New `unsafe_fromUserQuery` invocation found. This is not safe. Please use one of the PromptString helpers instead.
);
}

const resultsResp = await graphqlClient.contextSearchEvalDebug({
repoIDs,
query,
filePatterns: [],
codeResultsCount: clientOpts.codeResultsCount,
textResultsCount: clientOpts.textResultsCount,
});

if (isError(resultsResp)) {
throw new Error(
`contextSearch failed for repos [${repoNames.join(',')}] and query "${query}": ${resultsResp}`
);
}
if (resultsResp === null) {
throw new Error(
`contextSearch failed for repos [${repoNames.join(',')}] and query "${query}": null results`
);
}

const results = resultsResp ?? [];
const actualContext: EvalContextItem[] = [];
for (const contextList of results) {
actualContext.push(
...contextList.contextList.map(result => ({
repoName: result.repoName.replace(/^github\.com\//, ''),
path: result.path,
startLine: result.startLine,
endLine: result.endLine,
content: result.content,
retriever: contextList.name,
}))
);
}
exampleOutputs.push({
...example,
actualContext,
});

success = true;
} catch (error) {
console.error(`Attempt ${attempt + 1} failed: ${error.message}`);

Check failure on line 241 in agent/src/cli/command-bench/strategy-chat-context.ts

View workflow job for this annotation

GitHub Actions / test-unit (ubuntu, 20) 

'error' is of type 'unknown'.

Check failure on line 241 in agent/src/cli/command-bench/strategy-chat-context.ts

View workflow job for this annotation

GitHub Actions / test-unit (ubuntu, 18) 

'error' is of type 'unknown'.

Check failure on line 241 in agent/src/cli/command-bench/strategy-chat-context.ts

View workflow job for this annotation

GitHub Actions / JetBrains tests 

'error' is of type 'unknown'.

Check failure on line 241 in agent/src/cli/command-bench/strategy-chat-context.ts

View workflow job for this annotation

GitHub Actions / test-unit (windows, 20) 

'error' is of type 'unknown'.
if (attempt < 2) {
await sleep(1000);
}
attempt++;
}
}

const results = resultsResp ?? []
const actualContext: EvalContextItem[] = []
for (const contextList of results) {
actualContext.push(
...contextList.contextList.map(result => ({
repoName: result.repoName.replace(/^github\.com\//, ''),
path: result.path,
startLine: result.startLine,
endLine: result.endLine,
content: result.content,
retriever: contextList.name,
}))
)
if (!success) {
throw new Error(`Failed to process example after 3 attempts: ${JSON.stringify(example)}`);
}
exampleOutputs.push({
...example,
actualContext,
})
}

return exampleOutputs
}
await sleep(1000);
}
return exampleOutputs;
}
Loading