scrape-benin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from  util.spider import Spider\n",
    "from util.afrikcuisine import AfrikCuisine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attrs = {\n",
    "        'name'       :  'name',\n",
    "        'ingredients':  'recipeIngredient',\n",
    "        'total_time':   'totalTime',\n",
    "        'instructions': 'recipeInstructions',\n",
    "        'servings':     'recipeYield',\n",
    "        'category':     'recipeCategory',\n",
    "        'prep_time':    'prepTime',\n",
    "        'cook_time':    'cookTime',\n",
    "}\n",
    "listing={'items': '//div[@class=\"tile_content\"]/a[contains(@href,\"recettes\")]/@href', 'next': { 'next_page_str': '', 'type': 'url'}}\n",
    "seeds = ['https://www.cuisineaz.com/cuisine-du-monde/benin-p265']\n",
    "available_json = {'xpath' : \"normalize-space(//script[@type='application/ld+json'][contains(text(), 'recipeIngredient')])\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuisineaz_spider= Spider('https://www.cuisineaz.com', seeds= seeds, listing =listing,attrs= attrs, available_json=available_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuisineaz_spider.scrape_one_item('https://www.cuisineaz.com/recettes/igname-a-la-poele-88104.aspx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_list =cuisineaz_spider.start_scrape(max_pages=1,multithread=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(result_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/benin/cuisineaz.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'//span[@class=\"intertitre\" and contains(text(), \"Ingr\")]//text()'\n",
    "\n",
    "'//td//span[@class=\"intertitre\" and contains(text(), \"Description\")]/preceding-sibling::text()'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "attrs_2 = {\n",
    "    'name': 'normalize-space(//div[@class=\"titre\"]//text())',\n",
    "    'ingredients' : '//td//span[@class=\"intertitre\" and contains(text(), \"Préparation\")]/preceding-sibling::text()',\n",
    "    'total_time': '',\n",
    "    'servings': '//td//span[@class=\"intertitre\" and contains(text(), \"Préparation\")]/following-sibling::text()[preceding::span][1]',\n",
    "    'cook_time' : '//td//span[@class=\"intertitre\" and contains(text(), \"Préparation\")]/following-sibling::text()[preceding::span][3]',\n",
    "    'prep_time' : 'normalize-space(//td//span[@class=\"intertitre\" and contains(text(), \"Préparation\")]/following-sibling::text()[preceding::span])',\n",
    "    'instructions': '//td//span[@class=\"intertitre\" and contains(text(), \"Description\")]/following-sibling::p//text()',\n",
    "    'category':     '',    \n",
    "\n",
    "\n",
    "}\n",
    "seeds_2 = ['https://www.afrik-cuisine.com/recettes/benin/mot/', 'https://www.afrik-cuisine.com/recettes/benin/mot/?debut_une=10']\n",
    "listing_2={'items': '//div[@class=\"photo\"]/a/@href', 'next': { 'next_page_str': '', 'type': 'url'}}\n",
    "available_json_2={}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_header = { #setup custom header because romania requires certain headers\n",
    "        'referer': 'https://www.google.com/',\n",
    "        'Accept-Language': '*',\n",
    "        'Accept-Encoding': '*',\n",
    "        'Accept': '*',\n",
    "        'user-agent':''}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "afrik_spider= AfrikCuisine('https://www.afrik-cuisine.com/', seeds= seeds_2, listing =listing_2,attrs= attrs_2, available_json=available_json_2,header=custom_header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "afrik_spider.scrape_one_item('https://www.afrik-cuisine.com/recettes/fechouada')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "afrik_spider.scrape_one_item('https://www.afrik-cuisine.com/recettes/gari-foto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spider is scraping page: 1\n",
      "started chromedriver...\n",
      "found this much items:  10\n",
      "end of scrolling\n",
      "found this much items in the end:  10\n",
      "['recettes/calalou-dahomeen', 'recettes/la-sauce-man-tindjan', 'recettes/yovodoco-ou-yovo-doko', 'recettes/poulet-amiwo', 'recettes/omelette-a-la-banane', 'recettes/fontete', 'recettes/azinkokoui-au-boeuf', 'recettes/gari-dossi', 'recettes/gari-pinon', 'recettes/gari-foto']\n"
     ]
    },
    {
     "ename": "ConnectionError",
     "evalue": "HTTPSConnectionPool(host='www.afrik-cuisine.comrecettes', port=443): Max retries exceeded with url: /gari-dossi (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x12f13ca90>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mgaierror\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py:174\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=172'>173</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=173'>174</a>\u001b[0m     conn \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mcreate_connection(\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=174'>175</a>\u001b[0m         (\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dns_host, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mport), \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mextra_kw\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=175'>176</a>\u001b[0m     )\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=177'>178</a>\u001b[0m \u001b[39mexcept\u001b[39;00m SocketTimeout:\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/connection.py:72\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/connection.py?line=67'>68</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m six\u001b[39m.\u001b[39mraise_from(\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/connection.py?line=68'>69</a>\u001b[0m         LocationParseError(\u001b[39mu\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m, label empty or too long\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m host), \u001b[39mNone\u001b[39;00m\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/connection.py?line=69'>70</a>\u001b[0m     )\n\u001b[0;32m---> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/connection.py?line=71'>72</a>\u001b[0m \u001b[39mfor\u001b[39;00m res \u001b[39min\u001b[39;00m socket\u001b[39m.\u001b[39;49mgetaddrinfo(host, port, family, socket\u001b[39m.\u001b[39;49mSOCK_STREAM):\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/connection.py?line=72'>73</a>\u001b[0m     af, socktype, proto, canonname, sa \u001b[39m=\u001b[39m res\n",
      "File \u001b[0;32m/usr/local/Cellar/python@3.9/3.9.1_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/socket.py:953\u001b[0m, in \u001b[0;36mgetaddrinfo\u001b[0;34m(host, port, family, type, proto, flags)\u001b[0m\n\u001b[1;32m    <a href='file:///usr/local/Cellar/python%403.9/3.9.1_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/socket.py?line=951'>952</a>\u001b[0m addrlist \u001b[39m=\u001b[39m []\n\u001b[0;32m--> <a href='file:///usr/local/Cellar/python%403.9/3.9.1_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/socket.py?line=952'>953</a>\u001b[0m \u001b[39mfor\u001b[39;00m res \u001b[39min\u001b[39;00m _socket\u001b[39m.\u001b[39;49mgetaddrinfo(host, port, family, \u001b[39mtype\u001b[39;49m, proto, flags):\n\u001b[1;32m    <a href='file:///usr/local/Cellar/python%403.9/3.9.1_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/socket.py?line=953'>954</a>\u001b[0m     af, socktype, proto, canonname, sa \u001b[39m=\u001b[39m res\n",
      "\u001b[0;31mgaierror\u001b[0m: [Errno 8] nodename nor servname provided, or not known",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=701'>702</a>\u001b[0m \u001b[39m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=702'>703</a>\u001b[0m httplib_response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=703'>704</a>\u001b[0m     conn,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=704'>705</a>\u001b[0m     method,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=705'>706</a>\u001b[0m     url,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=706'>707</a>\u001b[0m     timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=707'>708</a>\u001b[0m     body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=708'>709</a>\u001b[0m     headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=709'>710</a>\u001b[0m     chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=710'>711</a>\u001b[0m )\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=712'>713</a>\u001b[0m \u001b[39m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=713'>714</a>\u001b[0m \u001b[39m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=714'>715</a>\u001b[0m \u001b[39m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=715'>716</a>\u001b[0m \u001b[39m# mess.\u001b[39;00m\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py:386\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=384'>385</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=385'>386</a>\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_validate_conn(conn)\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=386'>387</a>\u001b[0m \u001b[39mexcept\u001b[39;00m (SocketTimeout, BaseSSLError) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=387'>388</a>\u001b[0m     \u001b[39m# Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.\u001b[39;00m\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py:1040\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m   <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=1038'>1039</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mgetattr\u001b[39m(conn, \u001b[39m\"\u001b[39m\u001b[39msock\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m):  \u001b[39m# AppEngine might not have  `.sock`\u001b[39;00m\n\u001b[0;32m-> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=1039'>1040</a>\u001b[0m     conn\u001b[39m.\u001b[39;49mconnect()\n\u001b[1;32m   <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=1041'>1042</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m conn\u001b[39m.\u001b[39mis_verified:\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py:358\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=355'>356</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mconnect\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=356'>357</a>\u001b[0m     \u001b[39m# Add certificate verification\u001b[39;00m\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=357'>358</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msock \u001b[39m=\u001b[39m conn \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_new_conn()\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=358'>359</a>\u001b[0m     hostname \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhost\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py:186\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=184'>185</a>\u001b[0m \u001b[39mexcept\u001b[39;00m SocketError \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=185'>186</a>\u001b[0m     \u001b[39mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=186'>187</a>\u001b[0m         \u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mFailed to establish a new connection: \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m e\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=187'>188</a>\u001b[0m     )\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connection.py?line=189'>190</a>\u001b[0m \u001b[39mreturn\u001b[39;00m conn\n",
      "\u001b[0;31mNewConnectionError\u001b[0m: <urllib3.connection.HTTPSConnection object at 0x12f13ca90>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py:440\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=438'>439</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m chunked:\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=439'>440</a>\u001b[0m     resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=440'>441</a>\u001b[0m         method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=441'>442</a>\u001b[0m         url\u001b[39m=\u001b[39;49murl,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=442'>443</a>\u001b[0m         body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=443'>444</a>\u001b[0m         headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=444'>445</a>\u001b[0m         redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=445'>446</a>\u001b[0m         assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=446'>447</a>\u001b[0m         preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=447'>448</a>\u001b[0m         decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=448'>449</a>\u001b[0m         retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=449'>450</a>\u001b[0m         timeout\u001b[39m=\u001b[39;49mtimeout\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=450'>451</a>\u001b[0m     )\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=452'>453</a>\u001b[0m \u001b[39m# Send the request.\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=453'>454</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py:785\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=782'>783</a>\u001b[0m     e \u001b[39m=\u001b[39m ProtocolError(\u001b[39m\"\u001b[39m\u001b[39mConnection aborted.\u001b[39m\u001b[39m\"\u001b[39m, e)\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=784'>785</a>\u001b[0m retries \u001b[39m=\u001b[39m retries\u001b[39m.\u001b[39;49mincrement(\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=785'>786</a>\u001b[0m     method, url, error\u001b[39m=\u001b[39;49me, _pool\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m, _stacktrace\u001b[39m=\u001b[39;49msys\u001b[39m.\u001b[39;49mexc_info()[\u001b[39m2\u001b[39;49m]\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=786'>787</a>\u001b[0m )\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/connectionpool.py?line=787'>788</a>\u001b[0m retries\u001b[39m.\u001b[39msleep()\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/retry.py:592\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/retry.py?line=590'>591</a>\u001b[0m \u001b[39mif\u001b[39;00m new_retry\u001b[39m.\u001b[39mis_exhausted():\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/retry.py?line=591'>592</a>\u001b[0m     \u001b[39mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[39mor\u001b[39;00m ResponseError(cause))\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/urllib3/util/retry.py?line=593'>594</a>\u001b[0m log\u001b[39m.\u001b[39mdebug(\u001b[39m\"\u001b[39m\u001b[39mIncremented Retry for (url=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m): \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m\"\u001b[39m, url, new_retry)\n",
      "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='www.afrik-cuisine.comrecettes', port=443): Max retries exceeded with url: /gari-dossi (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x12f13ca90>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32m/Users/mahdiazhari/world-bank/scrape-benin.ipynb Cell 15'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/mahdiazhari/world-bank/scrape-benin.ipynb#ch0000014?line=0'>1</a>\u001b[0m result_list \u001b[39m=\u001b[39mafrik_spider\u001b[39m.\u001b[39;49mstart_scrape(max_pages\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m,multithread\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n",
      "File \u001b[0;32m~/world-bank/util/spider.py:352\u001b[0m, in \u001b[0;36mSpider.start_scrape\u001b[0;34m(self, max_pages, multithread)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=349'>350</a>\u001b[0m     results \u001b[39m=\u001b[39m []\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=350'>351</a>\u001b[0m     \u001b[39mfor\u001b[39;00m item_url \u001b[39min\u001b[39;00m listing_items:\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=351'>352</a>\u001b[0m         item \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mscrape_one_item(item_url)\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=352'>353</a>\u001b[0m         results\u001b[39m.\u001b[39mappend(item)\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=354'>355</a>\u001b[0m all_items \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m results\n",
      "File \u001b[0;32m~/world-bank/util/spider.py:59\u001b[0m, in \u001b[0;36mSpider.scrape_one_item\u001b[0;34m(self, url)\u001b[0m\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=56'>57</a>\u001b[0m \u001b[39m#add home page url to the obtained item url if item url is not complete\u001b[39;00m\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=57'>58</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmain_page \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m url:\n\u001b[0;32m---> <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=58'>59</a>\u001b[0m     res \u001b[39m=\u001b[39m session\u001b[39m.\u001b[39;49mget(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmain_page \u001b[39m+\u001b[39;49m url)\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=59'>60</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m     <a href='file:///Users/mahdiazhari/world-bank/util/spider.py?line=60'>61</a>\u001b[0m     res \u001b[39m=\u001b[39m session\u001b[39m.\u001b[39mget(url)\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py:542\u001b[0m, in \u001b[0;36mSession.get\u001b[0;34m(self, url, **kwargs)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=533'>534</a>\u001b[0m \u001b[39mr\u001b[39m\u001b[39m\"\"\"Sends a GET request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=534'>535</a>\u001b[0m \n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=535'>536</a>\u001b[0m \u001b[39m:param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=536'>537</a>\u001b[0m \u001b[39m:param \\*\\*kwargs: Optional arguments that ``request`` takes.\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=537'>538</a>\u001b[0m \u001b[39m:rtype: requests.Response\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=538'>539</a>\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=540'>541</a>\u001b[0m kwargs\u001b[39m.\u001b[39msetdefault(\u001b[39m'\u001b[39m\u001b[39mallow_redirects\u001b[39m\u001b[39m'\u001b[39m, \u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=541'>542</a>\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrequest(\u001b[39m'\u001b[39;49m\u001b[39mGET\u001b[39;49m\u001b[39m'\u001b[39;49m, url, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py:529\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=523'>524</a>\u001b[0m send_kwargs \u001b[39m=\u001b[39m {\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=524'>525</a>\u001b[0m     \u001b[39m'\u001b[39m\u001b[39mtimeout\u001b[39m\u001b[39m'\u001b[39m: timeout,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=525'>526</a>\u001b[0m     \u001b[39m'\u001b[39m\u001b[39mallow_redirects\u001b[39m\u001b[39m'\u001b[39m: allow_redirects,\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=526'>527</a>\u001b[0m }\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=527'>528</a>\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=528'>529</a>\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(prep, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49msend_kwargs)\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=530'>531</a>\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py:645\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=641'>642</a>\u001b[0m start \u001b[39m=\u001b[39m preferred_clock()\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=643'>644</a>\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=644'>645</a>\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39;49msend(request, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=646'>647</a>\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/sessions.py?line=647'>648</a>\u001b[0m elapsed \u001b[39m=\u001b[39m preferred_clock() \u001b[39m-\u001b[39m start\n",
      "File \u001b[0;32m~/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py:519\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=514'>515</a>\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(e\u001b[39m.\u001b[39mreason, _SSLError):\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=515'>516</a>\u001b[0m         \u001b[39m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=516'>517</a>\u001b[0m         \u001b[39mraise\u001b[39;00m SSLError(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[0;32m--> <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=518'>519</a>\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=520'>521</a>\u001b[0m \u001b[39mexcept\u001b[39;00m ClosedPoolError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    <a href='file:///Users/mahdiazhari/world-bank/wb-scraping/lib/python3.9/site-packages/requests/adapters.py?line=521'>522</a>\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m(e, request\u001b[39m=\u001b[39mrequest)\n",
      "\u001b[0;31mConnectionError\u001b[0m: HTTPSConnectionPool(host='www.afrik-cuisine.comrecettes', port=443): Max retries exceeded with url: /gari-dossi (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x12f13ca90>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known'))"
     ]
    }
   ],
   "source": [
    "result_list =afrik_spider.start_scrape(max_pages=1,multithread=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7f97af7b90ec3405e1f66aa9bfad4ba40234615df6d29df3ef268808d6a9feec"
  },
  "kernelspec": {
   "display_name": "Python 3.9.1 ('wb-scraping': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}