visualization/graphistry_cugraph/download_process_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff25a634-adb2-41d4-9855-6ef196ea4611",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install python-dotenv pydash yarl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47bf4f15",
   "metadata": {},
   "source": [
    "Packages in this code:\n",
    "\n",
    "* graphistry: 0.20.5\n",
    "* cugraph   : 21.10.0+0.g84617024.dirty\n",
    "* cudf      : 21.10.0\n",
    "* requests  : 2.26.0\n",
    "* pandas    : 1.3.3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52d3ef75-8425-4af7-9492-d7ae6f325562",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a6e21465-a3eb-4d15-a6a9-cc83713003d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "import graphistry\n",
    "import cudf\n",
    "import requests\n",
    "from pydash import py_\n",
    "import cudf\n",
    "import cugraph\n",
    "from yarl import URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5c1d0a90-a83e-43e0-b7f0-fa1f538c2b1f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "load_dotenv()\n",
    "DIFFBOT_TOKEN = os.getenv(\"DIFFBOT_TOKEN\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "774b1007-25a8-4b6f-8837-ad03957c4a97",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_file(query: str, size: int = 25, extract_data: bool = True):\n",
    "    \"\"\"Download data from Diffbot\"\"\"\n",
    "    query_string = f\"type=query&token={DIFFBOT_TOKEN}&query={query}&size={size}\"\n",
    "    url = URL.build(\n",
    "        scheme=\"https\",\n",
    "        host=\"kg.diffbot.com\",\n",
    "        path=\"/kg/dql_endpoint\",\n",
    "        query_string=query_string,\n",
    "    )\n",
    "    r = requests.get(url)\n",
    "    response = r.json()\n",
    "    return response[\"data\"] if extract_data else response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "904f233c-a8d6-403e-a0bf-8a6f8549b378",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = download_file(\n",
    "    'type:Person employments.{title:\"data scientist\" isCurrent:true} locations.country.name:\"United States of America\"',\n",
    "    size=28667,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3bfd9224-ffb0-40bc-a8d9-a03a8f617af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "\n",
    "# with open(\"data_scientist.json\", \"w\") as f:\n",
    "#     json.dump(data, f)\n",
    "\n",
    "# with open(\"data_scientist.json\", \"r\") as f:\n",
    "#     data = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67166949-b880-4160-bcaf-8966372364e6",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Process Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "17c73b74-49c9-4ed1-98b2-5825ef9705d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [py_(person).get(\"name\").value() for person in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "f2aa0009-ceb9-4864-bf67-3aa5cbc63d32",
   "metadata": {},
   "outputs": [],
   "source": [
    "employments = [\n",
    "    py_(person)\n",
    "    .get(\"employments\")\n",
    "    .filter({\"isCurrent\": True})\n",
    "    .filter(lambda x: \"Data Science\" in py_(x).get(\"categories\").map(\"name\").value())\n",
    "    .head()\n",
    "    .value()\n",
    "    for person in data\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "9fc7bd8e-86ac-4931-ad37-0d2ca2892755",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = [\n",
    "    py_(employment).get(\"categories\").map(\"name\").value() for employment in employments\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "f138720b-dd0f-44b3-bc10-6c9a1a28c7f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Engineering, IT and Software Development', 'Data Science']"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "4d4dffd1-94f5-4113-aea1-c9f565eb7930",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Chief Data Scientist'"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "py_(employments[0]).get(\"title\").value()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "8f9757fd-f629-43d1-8abd-80726816104c",
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = [py_(employment).get(\"title\").value() for employment in employments]\n",
    "titles = [title.lower() if title else title for title in titles]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "bde6856b-7958-42fb-9028-9b7824edb005",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "job_categories = set(py_.flatten(categories))\n",
    "levels = [\"Freelance\", \"Intern\", \"Senior\", \"Junior\", \"Student\"]\n",
    "roles = [\n",
    "    \"Backend Developer\",\n",
    "    \"Board Member\",\n",
    "    \"Chief Officer\",\n",
    "    \"DevOps Developer\",\n",
    "    \"Fullstack Developer or Architect\",\n",
    "    \"Director\",\n",
    "    \"Executive\",\n",
    "    \"Frontend Developer\",\n",
    "    \"Games Developer\",\n",
    "    \"Leadership\",\n",
    "    \"Manager\",\n",
    "    \"Mobile Developer\",\n",
    "    \"President\" \"Vice President\",\n",
    "    \"Management\",\n",
    "    \"Shareholder\",\n",
    "    \"Founder\",\n",
    "    \"CAO\",\n",
    "    \"CEO\",\n",
    "    \"CFO\",\n",
    "    \"CIO\",\n",
    "    \"CMO\",\n",
    "    \"COO\",\n",
    "    \"CTO\",\n",
    "    \"Chairman\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "76afab2a-5b67-4cdc-8c40-71708db3468a",
   "metadata": {},
   "outputs": [],
   "source": [
    "skills = [py_(person).get(\"skills\").map_(\"name\").value() for person in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "ee786f26-a544-4c00-a475-d6ca0124e6d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "domain = [\n",
    "    category\n",
    "    for category in job_categories\n",
    "    if category not in levels + roles and category\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "8dd6413c-13dc-4dc6-a8f6-3eb8d6b02bb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "83a43415-678a-4395-96cf-8a76a89473a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    {\"person\": names, \"skills\": skills, \"job_categories\": categories, \"titles\": titles}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "4918fe6e-d99e-47f5-89de-3d6f69c9ab32",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"level\"] = df[\"job_categories\"].apply(\n",
    "    lambda row: [category for category in row if category in levels]\n",
    ")\n",
    "\n",
    "df[\"role\"] = df[\"job_categories\"].apply(\n",
    "    lambda row: [category for category in row if category in roles]\n",
    ")\n",
    "\n",
    "df[\"domain\"] = df[\"job_categories\"].apply(\n",
    "    lambda row: [category for category in row if category in domain]\n",
    ")\n",
    "\n",
    "df = df.drop(columns=[\"job_categories\"]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "48bd28c0-5dd0-43d2-a282-eb303e282a78",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(\"data_scientist.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f93cffd2-10b8-41b7-963c-ae8f754d7b62",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}