02_data_cloud.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp data_cloud"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data_cloud\n",
    "\n",
    "> Read data from netdata cloud api into a pandas dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "# export\n",
    "import os\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "\n",
    "\n",
    "def get_data_cloud(space_id, room_id, context, after=-60, before=0, points=None, api_token=None, \n",
    "                   base_url='https://app.netdata.cloud', time_as_index=True, time_unit='ms',\n",
    "                   group='average', aggregations=[{'method': 'sum', 'groupBy': ['dimension']}],\n",
    "                   agent_options=['absolute', 'jsonwrap', 'nonzero', 'flip', 'ms'], gtime=0, with_metadata=True,\n",
    "                   format='array', node_ids=[], dimensions=[], freq=None\n",
    "                   ):\n",
    "    \"\"\"Get data from netdata cloud api.\n",
    "    \n",
    "    ##### Parameters:  \n",
    "    - **space_id** `str` The space_id you would like to pull data from.\n",
    "    - **room_id** `str` The room_id you would like to pull data from.\n",
    "    - **context** `str` The context you would like to pull data from.\n",
    "    - **after** `int` The timestamp or relative integer from which to pull data after.\n",
    "    - **before** `int` The timestamp or relative integer from which to pull data before.\n",
    "    - **points** `int` The `points` parameter to pass to the api call if need to aggregate data in some way.\n",
    "    - **api_token** `str` The api token to use for the api call.\n",
    "    - **base_url** `str` The base url to use for the api call.\n",
    "    - **time_as_index** `bool` Whether to set the time column as the index of the dataframe.\n",
    "    - **time_unit** `str` The unit of time to use for the time column.\n",
    "    - **group** `str` The `group` parameter to pass to the api call.\n",
    "    - **aggregations** `list` The `aggregations` parameter to pass to the api call.\n",
    "    - **agent_options** `list` The `agent_options` parameter to pass to the api call.\n",
    "    - **gtime** `int` The `gtime` parameter to pass to the api call.\n",
    "    - **with_metadata** `bool` The `with_metadata` parameter to pass to the api call.\n",
    "    - **format** `str` The `format` parameter to pass to the api call.\n",
    "    - **node_ids** `list` The `node_ids` parameter to pass to the api call.\n",
    "    - **dimensions** `list` The `dimensions` parameter to pass to the api call.\n",
    "    - **freq** `str` The frequency to use for the dataframe index.\n",
    "        \n",
    "    ##### Returns:  \n",
    "    - **df** `pd.DataFrame` A pandas dataframe.\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    if api_token is None:\n",
    "        api_token = os.getenv('NETDATA_API_TOKEN')\n",
    "        \n",
    "    window_length = before - after\n",
    "    if freq:\n",
    "        if freq.endswith('s'):\n",
    "            points = int(window_length / int(freq.replace('s','')))\n",
    "        elif freq.endswith('m'):\n",
    "            points = int(window_length / (int(freq.replace('m','')) * 60))\n",
    "        elif freq.endswith('h'):\n",
    "            points = int(window_length / (int(freq.replace('h','')) * 60 * 60))\n",
    "    \n",
    "    base_url = 'https://app.netdata.cloud'\n",
    "    url = f'{base_url}/api/v2/spaces/{space_id}/rooms/{room_id}/data'\n",
    "    headers = {'Accept': '*/*', 'Content-Type': 'application/json', 'Authorization': f'Bearer {api_token}'}\n",
    "    data = {\n",
    "        'filter': {\n",
    "            'nodeIDs': node_ids,\n",
    "            'context': context,\n",
    "            'dimensions': dimensions,\n",
    "        },\n",
    "        'after': int(after),\n",
    "        'before': int(before),\n",
    "        'format': format,\n",
    "        'group': group,\n",
    "        'aggregations': aggregations,\n",
    "        'agent_options': agent_options,\n",
    "        'gtime': gtime,\n",
    "        'with_metadata': with_metadata\n",
    "    }\n",
    "    if points:\n",
    "        data['points'] = points\n",
    "    r = requests.post(url, headers=headers, data=json.dumps(data))\n",
    "    \n",
    "    if r.status_code != 200:\n",
    "        \n",
    "        print(f'Error: {r.status_code, r.text}')\n",
    "        return pd.DataFrame()\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        df = pd.DataFrame(data=r.json()['result']['data'], columns=r.json()['result']['labels'])\n",
    "        \n",
    "        if time_unit:\n",
    "            df['time'] = pd.to_datetime(df['time'], unit='ms')\n",
    "        if time_as_index:\n",
    "            df = df.set_index('time')\n",
    "            \n",
    "        return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "# tests\n",
    "\n",
    "# load netdata api token from .env file\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "NETDATA_API_TOKEN = os.getenv('NETDATA_API_TOKEN')\n",
    "\n",
    "space_id = 'ea93d7b8-0df6-45c0-b13d-1560996c89eb'\n",
    "room_id = '9115ab73-317f-4a24-9233-f8d2993dbfc1'\n",
    "context = 'system.cpu'\n",
    "after = -60*10\n",
    "before = 0\n",
    "\n",
    "# get some charts from demo space\n",
    "df = get_data_cloud(space_id, room_id, context, after, before)\n",
    "\n",
    "# check df shape\n",
    "assert df.shape[0] >= 19\n",
    "assert df.shape[1] in [5,6]\n",
    "\n",
    "# get some charts from demo space\n",
    "df = get_data_cloud(space_id, room_id, context, after, before, freq='1s')\n",
    "\n",
    "# check df shape\n",
    "assert df.shape[0] >= 595\n",
    "assert df.shape[1] in [5,6]\n",
    "\n",
    "# get some charts from demo space\n",
    "df = get_data_cloud(space_id, room_id, context, after, before, freq='5m')\n",
    "\n",
    "# check df shape\n",
    "assert df.shape[0] <= 2\n",
    "assert df.shape[1] in [5,6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "\n",
    "\n",
    "def get_charts_cloud(space_id, room_id, api_token=None, base_url='https://app.netdata.cloud', node_ids=[]):\n",
    "    \"\"\"Get charts from netdata cloud api.\n",
    "    \"\"\"\n",
    "    \n",
    "    if api_token is None:\n",
    "        api_token = os.getenv('NETDATA_API_TOKEN')\n",
    "    \n",
    "    base_url = 'https://app.netdata.cloud'\n",
    "    url = f'{base_url}/api/v2/spaces/{space_id}/rooms/{room_id}/charts'\n",
    "    headers = {'Accept': '*/*', 'Content-Type': 'application/json', 'Authorization': f'Bearer {api_token}'}\n",
    "    data = {\n",
    "        'filter': {\n",
    "            'nodeIDs': node_ids,\n",
    "        }\n",
    "    }\n",
    "    r = requests.post(url, headers=headers, data=json.dumps(data))\n",
    "    \n",
    "    if r.status_code != 200:\n",
    "        \n",
    "        print(f'Error: {r.status_code, r.text}')\n",
    "        return None\n",
    "    \n",
    "    else:\n",
    "            \n",
    "        return r.json()['results']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "# tests\n",
    "\n",
    "# load netdata api token from .env file\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "NETDATA_API_TOKEN = os.getenv('NETDATA_API_TOKEN')\n",
    "\n",
    "space_id = 'ea93d7b8-0df6-45c0-b13d-1560996c89eb'\n",
    "room_id = '9115ab73-317f-4a24-9233-f8d2993dbfc1'\n",
    "\n",
    "# get charts from demo space\n",
    "charts = get_charts_cloud(space_id, room_id)\n",
    "assert 'system.cpu' in charts\n",
    "assert charts['system.cpu']['title'] == 'Total CPU utilization'\n",
    "assert charts['system.cpu']['units'] == 'percentage'\n",
    "assert charts['system.cpu']['priority'] == 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "bfe63f9ecba214a279dcde80da2d17bc123d05fb330a6a73beea2ff07f9df766"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}