Skip to content

Commit

Permalink
feat(middleware): process feed title and description using LLM (#16464)
Browse files Browse the repository at this point in the history 
* feat(middleware): add title translation using LLM

* refactor: combine title and description

* fix: camelCase for variables

* refactor: try to update the test case

---------
  • Loading branch information
@coxde
coxde authored Aug 21, 2024
1 parent d4bfda3 commit 2962c6f
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 61 deletions.
8 changes: 6 additions & 2 deletions lib/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ export type Config = {
temperature?: number;
maxTokens?: number;
endpoint: string;
prompt?: string;
inputOption: string;
promptTitle: string;
promptDescription: string;
};
bilibili: {
cookies: Record<string, string | undefined>;
Expand Down Expand Up @@ -436,7 +438,9 @@ const calculateValue = () => {
temperature: toInt(envs.OPENAI_TEMPERATURE, 0.2),
maxTokens: toInt(envs.OPENAI_MAX_TOKENS, 0) || undefined,
endpoint: envs.OPENAI_API_ENDPOINT || 'https://api.openai.com/v1',
prompt: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.',
inputOption: envs.OPENAI_INPUT_OPTION || 'description',
promptDescription: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.',
promptTitle: envs.OPENAI_PROMPT_TITLE || 'Please translate the following title into Simplified Chinese and reply only translated text.',
},

// Route-specific Configurations
Expand Down
27 changes: 22 additions & 5 deletions lib/middleware/parameter.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,8 @@ describe('multi parameter', () => {
});

describe('openai', () => {
it(`chatgpt`, async () => {
it('processes both title and description', async () => {
config.openai.inputOption = 'both';
const responseWithGpt = await app.request('/test/gpt?chatgpt=true');
const responseNormal = await app.request('/test/gpt');

Expand All @@ -437,9 +438,25 @@ describe('openai', () => {
const parsedGpt = await parser.parseString(await responseWithGpt.text());
const parsedNormal = await parser.parseString(await responseNormal.text());

expect(parsedGpt.items[0].content).not.toBe(undefined);
expect(parsedGpt.items[0].content).toBe(parsedNormal.items[0].content);
expect(parsedGpt.items[1].content).not.toBe(undefined);
expect(parsedGpt.items[1].content).not.toBe(parsedNormal.items[1].content);
expect(parsedGpt.items[0].title).not.toBe(parsedNormal.items[0].title);
expect(parsedGpt.items[0].title).toContain('AI processed content.');
expect(parsedGpt.items[0].content).not.toBe(parsedNormal.items[0].content);
expect(parsedGpt.items[0].content).toContain('AI processed content.');
});

it('processes title or description', async () => {
// test title
config.openai.inputOption = 'title';
const responseTitleOnly = await app.request('/test/gpt?chatgpt=true');
const parsedTitleOnly = await parser.parseString(await responseTitleOnly.text());
expect(parsedTitleOnly.items[0].title).toContain('AI processed content.');
expect(parsedTitleOnly.items[0].content).not.toContain('AI processed content.');

// test description
config.openai.inputOption = 'description';
const responseDescriptionOnly = await app.request('/test/gpt?chatgpt=true');
const parsedDescriptionOnly = await parser.parseString(await responseDescriptionOnly.text());
expect(parsedDescriptionOnly.items[0].title).not.toContain('AI processed content.');
expect(parsedDescriptionOnly.items[0].content).toContain('AI processed content.');
});
});
64 changes: 47 additions & 17 deletions lib/middleware/parameter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,16 @@ const resolveRelativeLink = ($: CheerioAPI, elem: Element, attr: string, baseUrl
}
};

const summarizeArticle = async (articleText: string) => {
const getAiCompletion = async (prompt: string, text: string) => {
const apiUrl = `${config.openai.endpoint}/chat/completions`;
const response = await ofetch(apiUrl, {
method: 'POST',
body: {
model: config.openai.model,
max_tokens: config.openai.maxTokens,
messages: [
{ role: 'system', content: config.openai.prompt },
{ role: 'user', content: articleText },
{ role: 'system', content: prompt },
{ role: 'user', content: text },
],
temperature: config.openai.temperature,
},
Expand Down Expand Up @@ -327,23 +327,53 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
if (ctx.req.query('chatgpt') && config.openai.apiKey) {
data.item = await Promise.all(
data.item.map(async (item) => {
if (item.description) {
try {
const summary = await cache.tryGet(`openai:${item.link}`, async () => {
const text = convert(item.description!);
if (text.length < 300) {
return '';
}
const summary_md = await summarizeArticle(text);
return md.render(summary_md);
try {
// handle description
if (config.openai.inputOption === 'description' && item.description) {
const description = await cache.tryGet(`openai:description:${item.link}`, async () => {
const description = convert(item.description!);
const descriptionMd = await getAiCompletion(config.openai.promptDescription, description);
return md.render(descriptionMd);
});
// 将总结结果添加到文章数据中
if (summary !== '') {
item.description = summary + '<hr/><br/>' + item.description;
// add it to the description
if (description !== '') {
item.description = description + '<hr/><br/>' + item.description;
}
}
// handle title
else if (config.openai.inputOption === 'title' && item.title) {
const title = await cache.tryGet(`openai:title:${item.link}`, async () => {
const title = convert(item.title!);
return await getAiCompletion(config.openai.promptTitle, title);
});
// replace the title
if (title !== '') {
item.title = title + '';
}
}
// handle both
else if (config.openai.inputOption === 'both' && item.title && item.description) {
const title = await cache.tryGet(`openai:title:${item.link}`, async () => {
const title = convert(item.title!);
return await getAiCompletion(config.openai.promptTitle, title);
});
// replace the title
if (title !== '') {
item.title = title + '';
}

const description = await cache.tryGet(`openai:description:${item.link}`, async () => {
const description = convert(item.description!);
const descriptionMd = await getAiCompletion(config.openai.promptDescription, description);
return md.render(descriptionMd);
});
// add it to the description
if (description !== '') {
item.description = description + '<hr/><br/>' + item.description;
}
} catch {
// when openai failed, return default description and not write cache
}
} catch {
// when openai failed, return default content and not write cache
}
return item;
})
Expand Down
42 changes: 6 additions & 36 deletions lib/routes/test/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,42 +299,12 @@ async function handler(ctx) {
break;

case 'gpt':
item.push(
{
title: 'Title0',
description: 'Description0',
pubDate: new Date(`2019-3-1`).toUTCString(),
link: 'https://github.com/DIYgod/RSSHub/issues/0',
},
{
title: 'Title1',
description:
'快速开始\n' +
'如果您在使用 RSSHub 过程中遇到了问题或者有建议改进，我们很乐意听取您的意见！您可以通过 Pull Request 来提交您的修改。无论您对 Pull Request 的使用是否熟悉，我们都欢迎不同经验水平的开发者参与贡献。如果您不懂编程，也可以通过 报告错误 的方式来帮助我们。\n' +
'\n' +
'参与讨论\n' +
'Telegram 群组 GitHub Issues GitHub 讨论\n' +
'\n' +
'开始之前\n' +
'要制作一个 RSS 订阅，您需要结合使用 Git、HTML、JavaScript、jQuery 和 Node.js。\n' +
'\n' +
'如果您对它们不是很了解，但想要学习它们，以下是一些好的资源：\n' +
'\n' +
'MDN Web Docs 上的 JavaScript 指南\n' +
'W3Schools\n' +
'Codecademy 上的 Git 课程\n' +
'如果您想查看其他开发人员如何使用这些技术来制作 RSS 订阅的示例，您可以查看 我们的代码库 中的一些代码。\n' +
'\n' +
'提交新的 RSSHub 规则\n' +
'如果您发现一个网站没有提供 RSS 订阅，您可以使用 RSSHub 制作一个 RSS 规则。RSS 规则是一个短小的 Node.js 程序代码（以下简称 “路由”），它告诉 RSSHub 如何从网站中提取内容并生成 RSS 订阅。通过制作新的 RSS 路由，您可以帮助让您喜爱的网站的内容被更容易访问和关注。\n' +
'\n' +
'在您开始编写 RSS 路由之前，请确保源站点没有提供 RSS。一些网页会在 HTML 头部中包含一个 type 为 application/atom+xml 或 application/rss+xml 的 link 元素来指示 RSS 链接。\n' +
'\n' +
'这是在 HTML 头部中看到 RSS 链接可能会长成这样：<link rel="alternate" type="application/rss+xml" href="http://example.com/rss.xml" />。如果您看到这样的链接，这意味着这个网站已经有了一个 RSS 订阅，您不需要为它制作一个新的 RSS 路由。',
pubDate: new Date(`2019-3-1`).toUTCString(),
link: 'https://github.com/DIYgod/RSSHub/issues/1',
}
);
item.push({
title: 'Title0',
description: 'Description0',
pubDate: new Date(`2019-3-1`).toUTCString(),
link: 'https://github.com/DIYgod/RSSHub/issues/0',
});

break;

Expand Down
2 changes: 1 addition & 1 deletion lib/setup.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const server = setupServer(
choices: [
{
message: {
content: 'Summary of the article.',
content: 'AI processed content.',
},
},
],
Expand Down

0 comments on commit 2962c6f

Please sign in to comment.