From a3474661d22f9107ead0303a7375452b8a6639e3 Mon Sep 17 00:00:00 2001 From: sk Date: Mon, 6 Jan 2025 01:34:25 +0100 Subject: [PATCH 1/7] add genossenschaften.immo --- lib/routes/genossenschaftenimmo/index.ts | 117 +++++++++++++++++++ lib/routes/genossenschaftenimmo/namespace.ts | 8 ++ 2 files changed, 125 insertions(+) create mode 100644 lib/routes/genossenschaftenimmo/index.ts create mode 100644 lib/routes/genossenschaftenimmo/namespace.ts diff --git a/lib/routes/genossenschaftenimmo/index.ts b/lib/routes/genossenschaftenimmo/index.ts new file mode 100644 index 00000000000000..55a66577ecc78e --- /dev/null +++ b/lib/routes/genossenschaftenimmo/index.ts @@ -0,0 +1,117 @@ +import { load } from 'cheerio'; +import ofetch from '@/utils/ofetch'; +import type { Data, DataItem, Route } from '@/types'; + +const FEED_TITLE = 'Genossenschaften.immo'; +const FEED_LOGO = 'https://genossenschaften.immo/static/gimmo/img/favicon/favicon-128x128.png'; +const FEED_LANGUAGE = 'de'; +const BASE_URL = 'https://genossenschaften.immo'; + +export const route: Route = { + name: 'Immobiliensuche', + path: '*', + maintainers: ['sk22'], + categories: ['other'], + description: ` +Note that all parameters are optional and many can be specified multiple times +(e.g. \`&district=wien-1-innere-stadt&district=wien-2-leopoldstadt\`). + +Only returns the first page of search results, allowing you to keep track of +newly added apartments. If you're looking for an apartment, make sure to also +look through the other pages on the website. + +:::tip +To get your query URL, go to https://genossenschaften.immo, open your browser's +dev tools (F12 or Ctrl+Shift+I) and go to the Network tab and filter for +XHR/Fetch requests. On the website, set up your search parameters. As the +results refresh, you'll see new requests to https://genossenschaften.immo/?… +coming in. Copy everything starting with the \`?\` to the end of the URL. +::: +`, + example: + '/genossenschaftenimmo' + + '?district=wien-1-innere-stadt&district=wien-2-leopoldstadt&district=wien-3-landstrasse&district=wien-4-wieden&district=wien-5-margareten&district=wien-6-mariahilf&district=wien-7-neubau&district=wien-8-josefstadt&district=wien-9-alsergrund&district=wien-10-favoriten&district=wien-11-simmering&district=wien-12-meidling&district=wien-13-hietzing&district=wien-14-penzing&district=wien-15-rudolfsheim-fuenfhaus&district=wien-16-ottakring&district=wien-17-hernals&district=wien-18-waehring&district=wien-19-doebling&district=wien-20-brigittenau&district=wien-21-floridsdorf&district=wien-22-donaustadt&district=wien-23-liesing' + + '&has_rent=on&has_rent_option=on' + + '&status=available&status=construction' + + '&cost=1000&room=2&size=60' + + '&has_property=off&has_rent=on&has_rent_option=on' + + '&status=available&status=construction&status=planned' + + '&type=residence&type=project' + + '&keywords=text', + parameters: { + // labels are in german language because it's the same on the website + cost: 'Miete bis (in €, number)', + district: 'Bezirk (string, multiple)', + size: 'Größe ab (in m², number)', + room: 'Zimmer ab (number)', + genossenschaft: 'Bauvereinigung (string, multiple)', + own_funds: 'Eigenkapital bis', + has_property: 'Eigentum (`on` | `off`)', + has_rent: 'Miete (`on` | `off`)', + has_rent_option: 'Miete mit Kaufoption (`on` | `off`)', + status: 'multiple, `available` | `construction` | `planned`', + type: 'multiple, `residence` | `project`', + keywords: 'Keyword search', + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + async handler(ctx) { + // `?cost=1000&district=wien-1-innere-stadt&…` + const { search } = new URL(ctx.req.url); + // path might be something like `/genossenschaftenimmo/immobilien/regionen/wien` + const path = ctx.req.path.slice('/genossenschaftenimmo/'.length); + const link = `${BASE_URL}/${path}${search}`; + const response = await ofetch(link); + const $ = load(response); + + const items = $('[itemtype="https://schema.org/Apartment"]') + .toArray() + .map((el) => { + const $el = $(el); + const name = $el.find('[itemprop=name]').text(); + const body = $el.find('.card-body').text().trim(); + const itemLink = BASE_URL + $el.find('[itemprop=url]').attr('href'); + const itemImage = $el.find('[itemprop=image]').attr('href'); + const itemPubDate = $el.find('time').attr('datetime'); + + // e.g. ['2', '60,00 m²', '500,00 €', '10.000,00 €'] + const numbers = $el + .find('.card-body .fs-5') + .toArray() + .map((el) => $(el).text().trim()); + + // e.g. ['Verfügbar', 'OEVW', 'Miete', 'Wohneinheit'] + const itemCategories = $el + .find('.badge') + .toArray() + .map((el) => $(el).text().trim()); + + const titleAppendix = numbers.length ? ` | ${numbers.join(' · ')}` : ''; + const itemTitle = name + titleAppendix; + const itemDescription = itemCategories.join(' · ') + (body.length ? ` / ${body} · ` : ''); + + return { + title: itemTitle, + link: itemLink, + pubDate: itemPubDate, + description: itemDescription, + category: itemCategories, + image: itemImage, + } satisfies DataItem; + }); + + return { + title: FEED_TITLE, + language: FEED_LANGUAGE, + logo: FEED_LOGO, + item: items, + link, + } satisfies Data; + }, +}; diff --git a/lib/routes/genossenschaftenimmo/namespace.ts b/lib/routes/genossenschaftenimmo/namespace.ts new file mode 100644 index 00000000000000..5f6674be0602fa --- /dev/null +++ b/lib/routes/genossenschaftenimmo/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'Genossenschaften.immo', + url: 'genossenschaften.immo', + description: 'Search engine for Genossenschaft housing in Austria', + lang: 'de', +}; From b2198c905b220c510fb6cff51fcda2bfea641992 Mon Sep 17 00:00:00 2001 From: sk Date: Mon, 6 Jan 2025 01:53:08 +0100 Subject: [PATCH 2/7] convert austrian time zone to utc --- lib/routes/genossenschaftenimmo/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/routes/genossenschaftenimmo/index.ts b/lib/routes/genossenschaftenimmo/index.ts index 55a66577ecc78e..e8e8d54c6ed82b 100644 --- a/lib/routes/genossenschaftenimmo/index.ts +++ b/lib/routes/genossenschaftenimmo/index.ts @@ -1,6 +1,10 @@ import { load } from 'cheerio'; import ofetch from '@/utils/ofetch'; import type { Data, DataItem, Route } from '@/types'; +import dayjs from 'dayjs'; +import timezone from 'dayjs/plugin/timezone'; + +dayjs.extend(timezone); const FEED_TITLE = 'Genossenschaften.immo'; const FEED_LOGO = 'https://genossenschaften.immo/static/gimmo/img/favicon/favicon-128x128.png'; @@ -76,9 +80,10 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. const $el = $(el); const name = $el.find('[itemprop=name]').text(); const body = $el.find('.card-body').text().trim(); + const dateTime = $el.find('time').attr('datetime'); const itemLink = BASE_URL + $el.find('[itemprop=url]').attr('href'); const itemImage = $el.find('[itemprop=image]').attr('href'); - const itemPubDate = $el.find('time').attr('datetime'); + const itemPubDate = dayjs.tz(dateTime, 'Europe/Vienna').toISOString(); // e.g. ['2', '60,00 m²', '500,00 €', '10.000,00 €'] const numbers = $el From 115ca653bea7da5c53bc03a490a4d41332aec3c2 Mon Sep 17 00:00:00 2001 From: sk Date: Mon, 6 Jan 2025 01:58:01 +0100 Subject: [PATCH 3/7] fix formatting error --- lib/routes/genossenschaftenimmo/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/genossenschaftenimmo/index.ts b/lib/routes/genossenschaftenimmo/index.ts index e8e8d54c6ed82b..a8136f962b96b7 100644 --- a/lib/routes/genossenschaftenimmo/index.ts +++ b/lib/routes/genossenschaftenimmo/index.ts @@ -99,7 +99,7 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. const titleAppendix = numbers.length ? ` | ${numbers.join(' · ')}` : ''; const itemTitle = name + titleAppendix; - const itemDescription = itemCategories.join(' · ') + (body.length ? ` / ${body} · ` : ''); + const itemDescription = itemCategories.join(' · ') + (body.length ? ` / ${body}` : ''); return { title: itemTitle, From 36cfc9385a764711b8558695ec0560e99d8918b6 Mon Sep 17 00:00:00 2001 From: sk Date: Mon, 6 Jan 2025 02:15:55 +0100 Subject: [PATCH 4/7] allow empty; change example url --- lib/routes/genossenschaftenimmo/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/routes/genossenschaftenimmo/index.ts b/lib/routes/genossenschaftenimmo/index.ts index a8136f962b96b7..5e53a215fb904c 100644 --- a/lib/routes/genossenschaftenimmo/index.ts +++ b/lib/routes/genossenschaftenimmo/index.ts @@ -37,11 +37,10 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. '?district=wien-1-innere-stadt&district=wien-2-leopoldstadt&district=wien-3-landstrasse&district=wien-4-wieden&district=wien-5-margareten&district=wien-6-mariahilf&district=wien-7-neubau&district=wien-8-josefstadt&district=wien-9-alsergrund&district=wien-10-favoriten&district=wien-11-simmering&district=wien-12-meidling&district=wien-13-hietzing&district=wien-14-penzing&district=wien-15-rudolfsheim-fuenfhaus&district=wien-16-ottakring&district=wien-17-hernals&district=wien-18-waehring&district=wien-19-doebling&district=wien-20-brigittenau&district=wien-21-floridsdorf&district=wien-22-donaustadt&district=wien-23-liesing' + '&has_rent=on&has_rent_option=on' + '&status=available&status=construction' + - '&cost=1000&room=2&size=60' + + '&cost=1000&room=2&size=50' + '&has_property=off&has_rent=on&has_rent_option=on' + '&status=available&status=construction&status=planned' + - '&type=residence&type=project' + - '&keywords=text', + '&type=residence&type=project', parameters: { // labels are in german language because it's the same on the website cost: 'Miete bis (in €, number)', @@ -115,6 +114,7 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. title: FEED_TITLE, language: FEED_LANGUAGE, logo: FEED_LOGO, + allowEmpty: true, item: items, link, } satisfies Data; From e0e7d65b85c514ec170d0eddc3433efe9e69fc91 Mon Sep 17 00:00:00 2001 From: sk Date: Mon, 6 Jan 2025 02:16:56 +0100 Subject: [PATCH 5/7] add utc plugin --- lib/routes/genossenschaftenimmo/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/routes/genossenschaftenimmo/index.ts b/lib/routes/genossenschaftenimmo/index.ts index 5e53a215fb904c..4b59682a00717a 100644 --- a/lib/routes/genossenschaftenimmo/index.ts +++ b/lib/routes/genossenschaftenimmo/index.ts @@ -3,8 +3,10 @@ import ofetch from '@/utils/ofetch'; import type { Data, DataItem, Route } from '@/types'; import dayjs from 'dayjs'; import timezone from 'dayjs/plugin/timezone'; +import utc from 'dayjs/plugin/utc'; dayjs.extend(timezone); +dayjs.extend(utc); const FEED_TITLE = 'Genossenschaften.immo'; const FEED_LOGO = 'https://genossenschaften.immo/static/gimmo/img/favicon/favicon-128x128.png'; From 07c51395eb1e7a8664a57c0578bc6ccf607431d4 Mon Sep 17 00:00:00 2001 From: sk Date: Wed, 8 Jan 2025 00:46:22 +0100 Subject: [PATCH 6/7] rename to genossenschaften --- .../{genossenschaftenimmo => genossenschaften}/index.ts | 6 +++--- .../{genossenschaftenimmo => genossenschaften}/namespace.ts | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename lib/routes/{genossenschaftenimmo => genossenschaften}/index.ts (96%) rename lib/routes/{genossenschaftenimmo => genossenschaften}/namespace.ts (100%) diff --git a/lib/routes/genossenschaftenimmo/index.ts b/lib/routes/genossenschaften/index.ts similarity index 96% rename from lib/routes/genossenschaftenimmo/index.ts rename to lib/routes/genossenschaften/index.ts index 4b59682a00717a..0afbf745d64daa 100644 --- a/lib/routes/genossenschaftenimmo/index.ts +++ b/lib/routes/genossenschaften/index.ts @@ -35,7 +35,7 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. ::: `, example: - '/genossenschaftenimmo' + + '/genossenschaften' + '?district=wien-1-innere-stadt&district=wien-2-leopoldstadt&district=wien-3-landstrasse&district=wien-4-wieden&district=wien-5-margareten&district=wien-6-mariahilf&district=wien-7-neubau&district=wien-8-josefstadt&district=wien-9-alsergrund&district=wien-10-favoriten&district=wien-11-simmering&district=wien-12-meidling&district=wien-13-hietzing&district=wien-14-penzing&district=wien-15-rudolfsheim-fuenfhaus&district=wien-16-ottakring&district=wien-17-hernals&district=wien-18-waehring&district=wien-19-doebling&district=wien-20-brigittenau&district=wien-21-floridsdorf&district=wien-22-donaustadt&district=wien-23-liesing' + '&has_rent=on&has_rent_option=on' + '&status=available&status=construction' + @@ -69,8 +69,8 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. async handler(ctx) { // `?cost=1000&district=wien-1-innere-stadt&…` const { search } = new URL(ctx.req.url); - // path might be something like `/genossenschaftenimmo/immobilien/regionen/wien` - const path = ctx.req.path.slice('/genossenschaftenimmo/'.length); + // path might be something like `/genossenschaften/immobilien/regionen/wien` + const path = ctx.req.path.slice('/genossenschaften/'.length); const link = `${BASE_URL}/${path}${search}`; const response = await ofetch(link); const $ = load(response); diff --git a/lib/routes/genossenschaftenimmo/namespace.ts b/lib/routes/genossenschaften/namespace.ts similarity index 100% rename from lib/routes/genossenschaftenimmo/namespace.ts rename to lib/routes/genossenschaften/namespace.ts From 401bdf574aa5968d62d8111c378a675e50aa51a8 Mon Sep 17 00:00:00 2001 From: sk Date: Wed, 8 Jan 2025 01:07:44 +0100 Subject: [PATCH 7/7] use path instead of search query --- lib/routes/genossenschaften/index.ts | 29 +++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/lib/routes/genossenschaften/index.ts b/lib/routes/genossenschaften/index.ts index 0afbf745d64daa..a721a679946340 100644 --- a/lib/routes/genossenschaften/index.ts +++ b/lib/routes/genossenschaften/index.ts @@ -8,10 +8,11 @@ import utc from 'dayjs/plugin/utc'; dayjs.extend(timezone); dayjs.extend(utc); -const FEED_TITLE = 'Genossenschaften.immo'; -const FEED_LOGO = 'https://genossenschaften.immo/static/gimmo/img/favicon/favicon-128x128.png'; -const FEED_LANGUAGE = 'de'; -const BASE_URL = 'https://genossenschaften.immo'; +const FEED_TITLE = 'Genossenschaften.immo' as const; +const FEED_LOGO = 'https://genossenschaften.immo/static/gimmo/img/favicon/favicon-128x128.png' as const; +const FEED_LANGUAGE = 'de' as const; +const BASE_URL = 'https://genossenschaften.immo' as const; +const PATH_PREFIX = '/genossenschaften/' as const; export const route: Route = { name: 'Immobiliensuche', @@ -20,7 +21,7 @@ export const route: Route = { categories: ['other'], description: ` Note that all parameters are optional and many can be specified multiple times -(e.g. \`&district=wien-1-innere-stadt&district=wien-2-leopoldstadt\`). +(e.g. \`district=wien-1-innere-stadt&district=wien-2-leopoldstadt\`). Only returns the first page of search results, allowing you to keep track of newly added apartments. If you're looking for an apartment, make sure to also @@ -31,12 +32,12 @@ To get your query URL, go to https://genossenschaften.immo, open your browser's dev tools (F12 or Ctrl+Shift+I) and go to the Network tab and filter for XHR/Fetch requests. On the website, set up your search parameters. As the results refresh, you'll see new requests to https://genossenschaften.immo/?… -coming in. Copy everything starting with the \`?\` to the end of the URL. +coming in. Copy everything after the \`?\` until the end of the URL. ::: `, example: - '/genossenschaften' + - '?district=wien-1-innere-stadt&district=wien-2-leopoldstadt&district=wien-3-landstrasse&district=wien-4-wieden&district=wien-5-margareten&district=wien-6-mariahilf&district=wien-7-neubau&district=wien-8-josefstadt&district=wien-9-alsergrund&district=wien-10-favoriten&district=wien-11-simmering&district=wien-12-meidling&district=wien-13-hietzing&district=wien-14-penzing&district=wien-15-rudolfsheim-fuenfhaus&district=wien-16-ottakring&district=wien-17-hernals&district=wien-18-waehring&district=wien-19-doebling&district=wien-20-brigittenau&district=wien-21-floridsdorf&district=wien-22-donaustadt&district=wien-23-liesing' + + PATH_PREFIX + + 'district=wien-1-innere-stadt&district=wien-2-leopoldstadt&district=wien-3-landstrasse&district=wien-4-wieden&district=wien-5-margareten&district=wien-6-mariahilf&district=wien-7-neubau&district=wien-8-josefstadt&district=wien-9-alsergrund&district=wien-10-favoriten&district=wien-11-simmering&district=wien-12-meidling&district=wien-13-hietzing&district=wien-14-penzing&district=wien-15-rudolfsheim-fuenfhaus&district=wien-16-ottakring&district=wien-17-hernals&district=wien-18-waehring&district=wien-19-doebling&district=wien-20-brigittenau&district=wien-21-floridsdorf&district=wien-22-donaustadt&district=wien-23-liesing' + '&has_rent=on&has_rent_option=on' + '&status=available&status=construction' + '&cost=1000&room=2&size=50' + @@ -67,11 +68,13 @@ coming in. Copy everything starting with the \`?\` to the end of the URL. supportScihub: false, }, async handler(ctx) { - // `?cost=1000&district=wien-1-innere-stadt&…` - const { search } = new URL(ctx.req.url); - // path might be something like `/genossenschaften/immobilien/regionen/wien` - const path = ctx.req.path.slice('/genossenschaften/'.length); - const link = `${BASE_URL}/${path}${search}`; + let path = ctx.req.path.slice(PATH_PREFIX.length); + if (path.startsWith('&')) { + // in case request url is something like `/genossenschaften/&cost=…` + path = path.slice(1); + } + + const link = `${BASE_URL}/?${path}`; const response = await ofetch(link); const $ = load(response);