scrape-moldova.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from util.onethousandmenu import OneThousandMenu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "attrs = {\n",
    "        'name'       :  '//h1[@itemprop=\"name\"]//text()',\n",
    "        'ingredients':  '//meta[@itemprop=\"recipeIngredient\"]/@content',\n",
    "        'total_time':   '//meta[@itemprop=\"totalTime\"]/@content',\n",
    "        'instructions': '//ol[@itemprop=\"recipeInstructions\"]//li//p[@class=\"instruction\"]//text()',\n",
    "        'servings':     'normalize-space(//meta[@itemprop=\"recipeYield\"]/@content)',\n",
    "        'category':     '',\n",
    "        'prep_time':    '',\n",
    "        'cook_time':    '',\n",
    "}\n",
    "listing={'items': '//div[@class=\"info-preview\"]//a[@class=\"h5\"]/@href', 'next': { 'next_page_str': '', 'type': 'url'}}\n",
    "seeds = ['https://1000.menu/catalog/moldavskaya-kuxnya']\n",
    "available_json = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user_agent = get_random_ua()\n",
    "custom_header = { #setup custom header because egypt requires certain headers\n",
    "        'referer': 'https://www.google.com/',\n",
    "        'Accept-Language': '*',\n",
    "        'Accept-Encoding': '*',\n",
    "        'Accept': '*',\n",
    "        'user-agent': ''}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "onekmenu_spider=  OneThousandMenu('https://1000.menu', seeds= seeds, listing =listing,attrs= attrs, available_json=available_json,header=custom_header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onekmenu_spider.scrape_one_item('https://1000.menu/cooking/43265-mititei-po-moldavski')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "started chromedriver...\n",
      "212\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/cooking/8764-klassicheskii-farshirovannyi-perec',\n",
       " '/cooking/27183-sup-chorba',\n",
       " '/cooking/43687-mamalyga-iz-kukuruznoi-krupy-po-moldavski',\n",
       " '/cooking/42443-placindy-moldavskie-s-tvorogom',\n",
       " '/cooking/43265-mititei-po-moldavski',\n",
       " '/cooking/860-shapka-gugutse---blinchiki-s-vishnei-v-smetannom-kreme',\n",
       " '/cooking/26718-verzere-moldavskie-pirojki',\n",
       " '/cooking/42438-placindy-moldavskie-s-tykvoi-v-duxovke',\n",
       " '/cooking/44341-kozunak-velikdenski-recept-pasxalnogo-kulicha-xleba',\n",
       " '/cooking/42437-moldavskie-vertuty-s-tykvoi-i-kapustoi',\n",
       " '/cooking/23049-vertuta-s-myasom',\n",
       " '/cooking/23045-vertuta-s-gribami',\n",
       " '/cooking/23043-vertuta-s-kartoshkoi',\n",
       " '/cooking/23039-vertuta-s-kapustoi',\n",
       " '/cooking/22324-sup-zama-moldavskaya',\n",
       " '/cooking/20623-pastorma-v-multivarke',\n",
       " '/cooking/20622-pastorma-v-duxovke',\n",
       " '/cooking/19876-placindy-na-kefire',\n",
       " '/cooking/19875-placindy-v-duxovke',\n",
       " '/cooking/19874-placinda-s-zelenu',\n",
       " '/cooking/19873-placindy-s-tykvoi',\n",
       " '/cooking/19872-placindy-s-kartoshkoi',\n",
       " '/cooking/10381-mamaljga-s-muzhdeem-i-zharenoi-rjboi',\n",
       " '/cooking/6593-zama',\n",
       " '/cooking/6383-moldavskie-pirozhki-s-kapustoi',\n",
       " '/cooking/6148-tokana',\n",
       " '/cooking/2283-borsh-zapravka',\n",
       " '/cooking/1748-sup-moldavskaya-zama-s-kuricei-i-ovoshchami',\n",
       " '/cooking/318-pechene-semilune',\n",
       " '/cooking/313-vertuta',\n",
       " '/cooking/290-platsindj-moldavskie',\n",
       " '/cooking/287-mamaljga',\n",
       " '/cooking/143-sup-s-govyadinoi-pomidorami-percem',\n",
       " '/cooking/44267-salat-iz-varenoi-morkovi-s-uksusom-chesnokom-i-petrushkoi',\n",
       " '/cooking/43809-borshch-so-svekloi-bolgarskim-percem-kapustoi-i-seldereem',\n",
       " '/cooking/34438-zakuska-iz-baklajan-na-zimu',\n",
       " '/cooking/15540-mititei-po-moldavski',\n",
       " '/cooking/11669-vertuti-s-tykvoj-iz-lavasha',\n",
       " '/cooking/4754-golubtsj-moldavskie--sarmale',\n",
       " '/cooking/2318-salat-moldova',\n",
       " '/cooking/1818-platsindj-s-tvorogom',\n",
       " '/cooking/795-baklazhanj-marinovannje-v-vine',\n",
       " '/cooking/325-mititei',\n",
       " '/cooking/48807-jele-iz-krasnoi-smorodiny-na-zimu',\n",
       " '/cooking/4311-malosolnje-ogurtsj-bez-rassola',\n",
       " '/cooking/13467-xrustyashchie-solenye-ogurcy-na-zimu-bez-uksusa',\n",
       " '/cooking/777-domashnie-solenye-ogurcy-xrustyashchie-v-bankax-na-zimu',\n",
       " '/cooking/7535-ovoshchnoe-assorti',\n",
       " '/cooking/323-klassicheskaya-okroshka-s-kolbasoi',\n",
       " '/cooking/21029-xolodnyi-borshch-klassicheskii-s-varenoi-svekloi-i-botvoi',\n",
       " '/cooking/1345-svekolnik-na-kefire-klassicheskii-xolodnyi-sup',\n",
       " '/cooking/22527-klassicheskaya-okroshka-na-kvase',\n",
       " '/cooking/13582-solenye-ogurcy-pod-kapronovoi-kryshkoi-na-zimu',\n",
       " '/cooking/46897-kulich-kraffin-pasxalnyi',\n",
       " '/cooking/13789-kabachki-po-koreiski-na-zimu',\n",
       " '/cooking/41674-pasxa-prostaya-tvorojnaya',\n",
       " '/cooking/18452-rassolnik-iz-svejix-ogurcov-s-perlovkoi',\n",
       " '/cooking/50176-pyshnaya-tvorojnaya-zapekanka-v-duxovke',\n",
       " '/cooking/809-pitstsa-domashnyaya',\n",
       " '/cooking/10055-zagotovka-rassolnika-na-zimu',\n",
       " '/cooking/8373-ogurchiki-po-koreiski-',\n",
       " '/cooking/13047-pasta-karbonara-s-bekonom-i-slivkami',\n",
       " '/cooking/39171-malosolnye-ogurcy-v-kastrule-xolodnym-sposobom',\n",
       " '/cooking/4235-lavash-jarenyi-s-yaicom',\n",
       " '/cooking/9905-tsarskoe-varene-iz-krjzhovnika',\n",
       " '/cooking/4584-tvorojnaya-zapekanka-kak-v-detskom-sadu',\n",
       " '/cooking/11599-sous-tartar',\n",
       " '/cooking/18655-myaso-po-francuzski-s-syrom-i-pomidorami-v-duxovke',\n",
       " '/cooking/44689-ostrye-ogurcy-na-zimu-marinovannye-s-uksusom',\n",
       " '/cooking/20384-kvas-iz-suxogo-kvasa-bez-drojjei',\n",
       " '/cooking/22330-kabachkovaya-ikra-kak-v-magazine-palchiki-oblijesh',\n",
       " '/cooking/10385-shaurma-s-kuricei-v-lavashe',\n",
       " '/cooking/33835-lenivye-xachapuri-s-syrom-na-skovorode-na-kefire',\n",
       " '/cooking/22013-kabachki-po-koreiski-bystrogo-prigotovleniya',\n",
       " '/cooking/13618-marinovannye-ogurcy-po-bolgarski-na-zimu',\n",
       " '/cooking/13122-myasnaya-sbornaya-solyanka-s-kartoshkoj',\n",
       " '/cooking/3943-kartofelnje-draniki',\n",
       " '/cooking/2030-salat-nejinskii-iz-ogurcov-na-zimu',\n",
       " '/cooking/44077-pasta-s-syrom',\n",
       " '/cooking/9926-shurpa-na-kostre',\n",
       " '/cooking/34324-klassicheskii-chaxoxbili-iz-kuricy-po-gruzinski',\n",
       " '/cooking/15886-gaspacho',\n",
       " '/cooking/9744-okroshka-na-mineralke',\n",
       " '/cooking/39197-goryachii-borsch-iz-marinovannoi-svekly',\n",
       " '/cooking/39256-korolevskaya-vatrushka-s-tvorogom-v-duxovke',\n",
       " '/cooking/22172-rolly-filadelfiya-v-domashnix-usloviyax',\n",
       " '/cooking/13306-xolodnyi-borshch-iz-marinovannoi-svekly-s-kolbasoi',\n",
       " '/cooking/3957-kartoshka-po-frantsuzski-s-myasom',\n",
       " '/cooking/42551-kazan-kebab-s-kartoshkoi-po-uzbekski-iz-baraniny',\n",
       " '/cooking/43907-salat-kurica-i-koreiskaya-morkov',\n",
       " '/cooking/17029-drojjevoe-testo-dlya-vatrushek',\n",
       " '/cooking/6190-pitstsa-pepperoni-',\n",
       " '/cooking/13606-zelenyi-sup-s-schavelem-yaicom-i-kuricei',\n",
       " '/cooking/9723-okroshka-na-maioneze',\n",
       " '/cooking/30395-lagman-iz-govyadiny',\n",
       " '/cooking/40317-domashnie-makarons',\n",
       " '/cooking/15611-pestraya-kartoshechka-s-lukom-i-morkovu',\n",
       " '/cooking/34010-aleksandriiskii-kulich-na-toplenom-moloke',\n",
       " '/cooking/19898-shurpa-iz-svininy-v-domashnix-usloviyax',\n",
       " '/cooking/44000-lazanya-klassicheskaya-s-farshem-v-duxovke',\n",
       " '/cooking/37261-udon-s-kuricei-i-ovoshchami',\n",
       " '/cooking/33777-kartoshka-s-myasom-v-kazane',\n",
       " '/cooking/45781-myaso-po-francuzski-iz-svininy-s-pomidorami',\n",
       " '/cooking/16699-lepeshki-s-zelenu-na-skovorode',\n",
       " '/cooking/13358-pankeiki-na-kefire-pyshnye',\n",
       " '/cooking/13747-ogurcy-kvashenye-v-bankax-na-zimu',\n",
       " '/cooking/17386-kvas-domashnii-iz-kvasnogo-susla',\n",
       " '/cooking/2602-pirojki-na-kefire-jarenye-na-skovorode',\n",
       " '/cooking/930-shchavelevji-sup',\n",
       " '/cooking/21175-azu-iz-govyadiny',\n",
       " '/cooking/30392-lapsha-dlya-lagmana-domashnyaya',\n",
       " '/cooking/10853-picca-margarita-v-domashnix-usloviyax',\n",
       " '/cooking/29375-kulesh-v-domashnix-usloviyax',\n",
       " '/cooking/14758-xachapuri-s-tvorogom-v-duxovke',\n",
       " '/cooking/9689-tkemali-iz-jeltoi-alychi-na-zimu',\n",
       " '/cooking/13586-xrustyashchie-marinovannye-ogurcy-s-limonnoi-kislotoi-na-zimu',\n",
       " '/cooking/13587-ogurcy-s-aspirinom-na-zimu',\n",
       " '/cooking/5233-solyanka-klassicheskaya-s-myasnym-naborom-i-maslinami-i-ogurcami',\n",
       " '/cooking/42558-lulya-kebab-v-duxovke-v-domashnix-usloviyax',\n",
       " '/cooking/43071-ruletiki-iz-baklajanov-s-greckimi-orexami',\n",
       " '/cooking/41792-pasxa-carskaya-zavarnaya',\n",
       " '/cooking/43048-domashnii-klassicheskii-tiramisu',\n",
       " '/cooking/9585-kvas-limonnyi',\n",
       " '/cooking/19030-lapsha-vok-s-kuricei-i-ovoshchami',\n",
       " '/cooking/36799-ratatui-v-duxovke-s-syrom',\n",
       " '/cooking/37472-francuzskoe-testo-domashnee',\n",
       " '/cooking/15111-klassicheskaya-dolma-v-vinogradnyx-listyax-v-kastrule',\n",
       " '/cooking/17258-ratatui-iz-baklajanov',\n",
       " '/cooking/666-morkov-po-koreiski-s-pripravoi',\n",
       " '/cooking/672-lobio-iz-krasnoi-fasoli',\n",
       " '/cooking/58363-zelenye-schi-s-krapivoi-i-schavelem-s-yaicom',\n",
       " '/cooking/1441-vareniki-s-chernikoi-na-kefire',\n",
       " '/cooking/16017-tursha-iz-fasoli-po-armyanski-na-zimu',\n",
       " '/cooking/12575-salat-nisuaz',\n",
       " '/cooking/48862-ogurcy-v-tomatnom-soke-na-zimu',\n",
       " '/cooking/17544-xvorost-klassicheskii',\n",
       " '/cooking/43965-salat-koul-slou-klassicheskii',\n",
       " '/cooking/1160-domashnii-kvas-iz-rjanogo-chernogo-xleba',\n",
       " '/cooking/39505-svinina-po-francuzski-s-kartofelem-v-duxovke',\n",
       " '/cooking/62244-tort-tiramisu-iz-pechenya-s-maskarpone-klassicheskii',\n",
       " '/cooking/32006-borshch-so-svejei-kapustoi-i-svekloi',\n",
       " '/cooking/44095-drojjevoe-testo-s-makom',\n",
       " '/cooking/16056-lagman-na-kostre-v-kazane',\n",
       " '/cooking/18048-rybnoe-seviche-iz-lososya',\n",
       " '/cooking/17756-befstroganov-klassicheskii',\n",
       " '/cooking/14007-drojjevye-pirojki-s-zelenym-lukom-i-yaicom-v-duxovke',\n",
       " '/cooking/14923-xachapuri-po-megrelski',\n",
       " '/cooking/49111-kvas-iz-rjanoi-muki-domashnii',\n",
       " '/cooking/39149-chashushuli-po-gruzinski-iz-govyadiny',\n",
       " '/cooking/34608-klassicheskaya-panna-kotta',\n",
       " '/cooking/9326-domashnie-xachapuri-s-syrom-na-kefire',\n",
       " '/cooking/18113-kak-pravilno-varit-kofe-v-turke',\n",
       " '/cooking/26915-kokteil-long-ailend',\n",
       " '/cooking/13897-drojjevoe-testo-dlya-belyashei-s-myasom',\n",
       " '/cooking/35845-sous-beshamel-klassicheskii',\n",
       " '/cooking/40709-salat-s-syrom-feta-i-pomidorami',\n",
       " '/cooking/42094-ais-kofe-po-vetnamski',\n",
       " '/cooking/8764-klassicheskii-farshirovannyi-perec',\n",
       " '/cooking/16008-ostrye-baklajany-na-zimu',\n",
       " '/cooking/15394-salat-chuka-s-orexovym-sousom',\n",
       " '/cooking/10738-adjika-iz-pomidor-na-zimu-bez-varki-jguchaya',\n",
       " '/cooking/29558-buuz',\n",
       " '/cooking/29449-buryatskie-buuzy-pozy-na-paru',\n",
       " '/cooking/22920-kurica-po-francuzski-v-duxovke',\n",
       " '/cooking/28984-ovoshchi-v-soevom-souse',\n",
       " '/cooking/26995-sup-xarcho-iz-svininy-s-risom',\n",
       " '/cooking/15961-julen-s-kuricei-i-gribami-na-skovorode',\n",
       " '/cooking/12013-ispanskie-churros',\n",
       " '/cooking/14064-picca-s-kuricei-i-ananasami',\n",
       " '/cooking/34164-litovskii-xolodnyi-borshch-na-kefire-iz-svekly',\n",
       " '/cooking/40399-fondan-shokoladnyi-s-jidkim-centrom',\n",
       " '/cooking/13550-domashnii-ketchup-iz-tomatnoi-pasty',\n",
       " '/cooking/9609-okroshka-na-smetane-i-vode',\n",
       " '/cooking/31142-cyplyonok-tabaka-v-duxovke',\n",
       " '/cooking/22521-sup-xarcho-klassicheskii-s-risom',\n",
       " '/cooking/10447-goroshnitsa',\n",
       " '/cooking/8139-pankeiki',\n",
       " '/cooking/15920-buiabes',\n",
       " '/cooking/2494-rollj-kaliforniya',\n",
       " '/cooking/36140-kutaby-s-zelenu-i-syrom',\n",
       " '/cooking/18353-govyadina-pastrami',\n",
       " '/cooking/48874-sous-pesto-klassicheskii-s-bazilikom',\n",
       " '/cooking/10167-samogon-iz-pshenitsj',\n",
       " '/cooking/29278-lamadjo-armyanskie-lepeshki',\n",
       " '/cooking/13846-vareniki-na-vode-s-vishnei',\n",
       " '/cooking/1655-goroxovyi-sup-na-myasnom-bulone-po-finski',\n",
       " '/cooking/40863-kitaiskii-kislo-sladkii-sous',\n",
       " '/cooking/20911-xychiny-s-kartoshkoi-i-syrom',\n",
       " '/cooking/28977-kolbasa-chorizo',\n",
       " '/cooking/46067-pasta-fetuchini-kurica-s-gribami-v-slivochnom-souse',\n",
       " '/cooking/42785-nejnaya-tvorojnaya-zapekanka-na-smetane-s-izumom',\n",
       " '/cooking/44041-kordon-blu-klassicheskii-s-vetchinoi-i-syrom',\n",
       " '/cooking/45359-sup-s-kleckami-na-kurinom-bulone',\n",
       " '/cooking/1731-kjstjbji-s-kartofelem',\n",
       " '/cooking/21199-forshmak-klassicheskii',\n",
       " '/cooking/17675-plov-v-kazane-iz-govyadiny',\n",
       " '/cooking/4979-domashnii-kofeinyi-liker-na-spirtu',\n",
       " '/cooking/34560-xarcho-iz-govyadiny-klassicheskii',\n",
       " '/cooking/64573-okroshka-na-kefire-s-mineralkoi',\n",
       " '/cooking/31689-kabachkovaya-ikra-klassicheskaya',\n",
       " '/cooking/36285-myaso-po-francuzski-iz-kurinogo-file-v-duxovke',\n",
       " '/cooking/16895-tartar-iz-govyadiny-klassicheskii',\n",
       " '/cooking/34576-ochen-lenivye-vkusnye-golubcy-na-skovorode',\n",
       " '/cooking/15419-italyanskaya-picca-4-syra-v-duxovke',\n",
       " '/cooking/24696-testo-na-suxom-moloke-na-pirojki',\n",
       " '/cooking/3991-kvas-bolotova',\n",
       " '/cooking/26783-sous-unagi-v-domashnix-usloviyax',\n",
       " '/cooking/1007-xachapuri-lodochka-po-adjarski-s-yaicom',\n",
       " '/cooking/38457-cyplenok-tabaka-na-skovorode-pod-pressom-klassicheskii',\n",
       " '/cooking/39917-klassicheskie-shchi-iz-svejei-kapusty',\n",
       " '/cooking/17510-xachapuri-s-tvorogom-i-syrom-na-skovorode',\n",
       " '/cooking/33135-korolevskie-krevetki-na-grile']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "onekmenu_spider.get_items_from_page('https://1000.menu/catalog/moldavskaya-kuxnya')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_list = onekmenu_spider.start_scrape(max_pages=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7f97af7b90ec3405e1f66aa9bfad4ba40234615df6d29df3ef268808d6a9feec"
  },
  "kernelspec": {
   "display_name": "Python 3.9.1 ('wb-scraping': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}