feat(add route): VisionIAS route for daily news (#18030)

* init

* fix __dir

* add news route

* not passing directly

* comment changes

---------

Co-authored-by: rjnishant530 <[email protected]>

lib/routes/visionias/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'VisionIAS',
+    url: 'visionias.in',
+    lang: 'en',
+    categories: ['study'],
+};

lib/routes/visionias/news-today.ts

@@ -0,0 +1,100 @@
+import { Data, Route } from '@/types';
+import { baseUrl, extractNews } from './utils';
+import dayjs from 'dayjs';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+import logger from '@/utils/logger';
+
+export const route: Route = {
+    path: '/newsToday/:filter?',
+    example: '/visionias/newsToday',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    parameters: {
+        filter: {
+            description: 'Period to fetch news for the current month. All news for the current month or only the latest',
+            default: 'latest',
+            options: [
+                { value: 'all', label: 'All' },
+                { value: 'latest', label: 'Latest' },
+            ],
+        },
+    },
+    radar: [
+        {
+            source: ['visionias.in/current-affairs/news-today'],
+            target: '/newsToday',
+        },
+    ],
+    name: 'News Today',
+    maintainers: ['Rjnishant530'],
+    handler,
+};
+
+async function handler(ctx): Promise<Data> {
+    const filter = ctx.req.param('filter') ?? 'latest';
+    const currentYear = dayjs().year();
+    const currentMonth = dayjs().month() + 1;
+    logger.debug(`Getting news for month ${currentMonth} and year ${currentYear}`);
+    const response = await ofetch(`${baseUrl}/current-affairs/news-today/getbymonth?year=${currentYear}&month=${currentMonth}`);
+
+    let items: any = [];
+    // let title = 'News Today';
+
+    if (response.length !== 0) {
+        if (filter === 'latest') {
+            const currentUrl = response[0].url;
+            // title = response[0].formatted_published_at;
+            items = await processCurrentNews(currentUrl);
+        } else {
+            const results = await Promise.all(response.map((element) => processCurrentNews(element.url)));
+            items.push(...results.flat());
+        }
+    }
+    return {
+        title: 'News Today | Current Affairs | Vision IAS',
+        link: `${baseUrl}/current-affairs/news-today/archive`,
+        description: 'News Today is a daily bulletin providing readers with a comprehensive overview of news developments, news types, and technical terms.',
+        language: 'en',
+        item: items,
+        image: `${baseUrl}/current-affairs/images/news-today-logo.svg`,
+        icon: `${baseUrl}/current-affairs/favicon.ico`,
+        logo: `${baseUrl}/current-affairs/favicon.ico`,
+        allowEmpty: true,
+    };
+}
+
+async function processCurrentNews(currentUrl) {
+    const response = await ofetch(`${baseUrl}${currentUrl}`);
+    const $ = load(response);
+    const items = $(`#table-of-content > ul > li > a`)
+        .toArray()
+        .map((item) => {
+            const link = $(item).attr('href');
+            const title = $(item).clone().children('span').remove().end().text().trim();
+            return {
+                title,
+                link: title === 'Also in News' ? link : `${baseUrl}${link}`,
+                guid: link,
+            };
+        });
+    const newsPromises = await Promise.allSettled(items.map((item) => extractNews(item, 'main > div > div.mt-6 > div.flex > div.flex.mt-6')));
+    const finalItems: any = [];
+    for (const news of newsPromises) {
+        if (news.status === 'fulfilled') {
+            finalItems.push(...(Array.isArray(news.value) ? news.value : [news.value]));
+        } else {
+            finalItems.push({
+                title: 'Error Parse News',
+            });
+        }
+    }
+    return finalItems;
+}

lib/routes/visionias/templates/description-sub.art

@@ -0,0 +1,6 @@
+{{ if heading }}
+  <h2>{{ heading }}</h2>
+{{ /if }}
+{{ if articleContent }}
+  {{@ articleContent }}
+{{ /if }}

lib/routes/visionias/templates/description.art

@@ -0,0 +1,12 @@
+{{ if heading }}
+  <h1>{{ heading }}</h1>
+{{ /if }}
+{{ if subItems }}
+  {{ each subItems item }}
+    {{ if item?.description }}
+       {{@ item.description }}
+    {{ /if }}
+  {{ /each }}
+{{else}}
+   {{@ articleContent }}
+{{ /if }}

lib/routes/visionias/utils.ts

@@ -0,0 +1,109 @@
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+import { DataItem } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+
+export const baseUrl = 'https://visionias.in';
+
+export async function extractNews(item, selector) {
+    if (item.link === '') {
+        return item;
+    }
+    return await cache.tryGet(item.link, async () => {
+        const response = await ofetch(item.link || '');
+        const $$ = load(response);
+        const postedDate = String($$('meta[property="article:published_time"]').attr('content'));
+        const updatedDate = String($$('meta[property="article:modified_time"]').attr('content'));
+        const tags = $$('meta[property="article:tag"]')
+            .toArray()
+            .map((tag) => $$(tag).attr('content'));
+        const content = $$(selector);
+        const heading = content.find('div.space-y-4 > h1').text();
+        const mainGroup = content.find('div.flex > div.w-full');
+
+        const shortArticles = mainGroup.find('[x-data^="{isShortArticleOpen"]');
+        const sections = mainGroup.find('[x-data^="{isSectionOpen"]');
+        if (shortArticles.length !== 0) {
+            const items = shortArticles.toArray().map((element) => {
+                const mainDiv = $$(element);
+                const title = mainDiv.find('a > div > h1').text().trim();
+                const id = mainDiv.find('a').attr('href');
+                const htmlContent = extractArticle(mainDiv.html());
+                const innerTags = mainDiv
+                    .find('ul > li:contains("Tags :")')
+                    ?.nextAll('li')
+                    .toArray()
+                    .map((tag) => $$(tag).text());
+                const description = art(path.join(__dirname, `templates/description.art`), {
+                    heading: title,
+                    articleContent: htmlContent,
+                });
+                return {
+                    title: `${title} | ${heading}`,
+                    pubDate: parseDate(postedDate),
+                    category: innerTags,
+                    description,
+                    link: `${item.link}${id}`,
+                    author: 'Vision IAS',
+                } as DataItem;
+            });
+            return items;
+        } else if (sections.length === 0) {
+            const htmlContent = extractArticle(mainGroup.html());
+            const description = art(path.join(__dirname, 'templates/description.art'), {
+                heading,
+                articleContent: htmlContent,
+            });
+            return {
+                title: item.title,
+                pubDate: parseDate(postedDate),
+                category: tags,
+                description,
+                link: item.link,
+                updated: updatedDate ? parseDate(updatedDate) : null,
+                author: 'Vision IAS',
+            } as DataItem;
+        } else {
+            const items = sections.toArray().map((element) => {
+                const mainDiv = $$(element);
+                const title = mainDiv.find('a > div > h2').text().trim();
+                const htmlContent = extractArticle(mainDiv.html(), 'div.ck-content');
+                const description = art(path.join(__dirname, `templates/description-sub.art`), {
+                    heading: title,
+                    articleContent: htmlContent,
+                });
+                return { description };
+            });
+            const description = art(path.join(__dirname, `templates/description.art`), {
+                heading,
+                subItems: items,
+            });
+            return {
+                title: heading,
+                pubDate: parseDate(postedDate),
+                category: tags,
+                description,
+                link: item.link,
+                updated: updatedDate ? parseDate(updatedDate) : null,
+                author: 'Vision IAS',
+            } as DataItem;
+        }
+    });
+}
+
+function extractArticle(articleDiv, selectorString: string = '#article-content') {
+    const $ = load(articleDiv, null, false);
+    const articleDiv$ = $(articleDiv);
+    const articleContent = articleDiv$.find(String(selectorString));
+    articleContent.find('figure').each((_, element) => {
+        $(element).css('width', '');
+    });
+    const htmlContent = articleContent.html();
+    return htmlContent;
+}

lib/routes/visionias/weekly-focus.ts

@@ -0,0 +1,56 @@
+import { Data, Route } from '@/types';
+import { baseUrl, extractNews } from './utils';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: '/weeklyFocus',
+    example: '/visionias/weeklyFocus',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['visionias.in/current-affairs/weekly-focus'],
+            target: '/weeklyFocus',
+        },
+    ],
+    name: 'Weekly Focus',
+    maintainers: ['Rjnishant530'],
+    handler,
+};
+
+async function handler(ctx): Promise<Data> {
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 1;
+    const response = await ofetch(`${baseUrl}/current-affairs/weekly-focus/archive`);
+    const $ = load(response);
+    const cards = $('div.weekly-focus-single-card').slice(0, limit).toArray();
+    const individualLinks = cards.flatMap((card) =>
+        $(card)
+            .find('a:has(p)')
+            .toArray()
+            .map((item) => {
+                const link = $(item).attr('href');
+                return { link: link?.startsWith('http') ? link : `${baseUrl}${link}` };
+            })
+    );
+
+    const itemsPromise = await Promise.allSettled(individualLinks.map(({ link }) => extractNews({ link }, 'main > div > div.flex > div.flex.w-full > div.w-full.mt-6')));
+
+    return {
+        title: 'Weekly Focus | Current Affairs | Vision IAS',
+        link: `${baseUrl}/current-affairs/weekly-focus/archive`,
+        description: 'Weekly Focus provides weekly comprehensive analysis of current themes with multidimensional and consolidated content.',
+        language: 'en',
+        item: itemsPromise.map((item) => (item.status === 'fulfilled' ? item.value : { title: 'Error Parse News' })),
+        image: `${baseUrl}/current-affairs/images/weekly-focus-logo.svg`,
+        icon: `${baseUrl}/current-affairs/favicon.ico`,
+        logo: `${baseUrl}/current-affairs/favicon.ico`,
+        allowEmpty: true,
+    };
+}