SecondRDFApproach.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "70be18e6-5e8a-441b-a6eb-21ecec6d396f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import rdflib \n",
    "from rdflib import URIRef, BNode, Literal, RDF, RDFS, Graph\n",
    "from rdflib.namespace import FOAF, XSD\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "217ac46b-2570-4519-b82b-c4a7d8e970b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "relation_file = 'panama_papers.edges.csv'\n",
    "\n",
    "nodes = {\n",
    "    'address': 'panama_papers.nodes.address.csv',\n",
    "    'entity': 'panama_papers.nodes.entity.csv',\n",
    "    'intermediary':'panama_papers.nodes.intermediary.csv',\n",
    "    'officer':'panama_papers.nodes.officer.csv'\n",
    "}\n",
    "\n",
    "output_file = './output/result.ttl'\n",
    "\n",
    "lbl_blank_node = 'blank_node_id'\n",
    "lbl_node_id = 'node_id'\n",
    "lbl_start_node = 'START_ID'\n",
    "lbl_end_node = 'END_ID'\n",
    "lbl_relation_type = 'TYPE'\n",
    "lbl_index='index'\n",
    "\n",
    "result2RDFStar=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "29830805-f583-4a66-92ca-445eab65e996",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = './input/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "78760cc2-e5c4-481f-9660-c3323a83c99d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-28 15:05:41.029903\n"
     ]
    }
   ],
   "source": [
    "start_time = datetime.now()\n",
    "print(start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "569e4517-81c4-46ef-89e2-9ad961d43be3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getRDFHeader(file_manager):\n",
    "    file_manager.write('@prefix ns1: <http://example.org/> .\\n')\n",
    "    file_manager.write('@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\\n\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "af0b6cd3-2aa4-44b5-ab52-02416253e4a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5e6a4854-2e2c-47c0-a34f-a32b9a1142a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture --no-display\n",
    "for n in nodes.keys():\n",
    "    dfs[n] = pd.read_csv(data_folder+nodes[n].format(n), sep=',', dtype='str')\n",
    "    for c in dfs[n]:\n",
    "        dfs[n][c] = dfs[n][c].str.replace('\"','').str.replace('\\\\','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8ed190b-fc4b-46b1-a4d9-f038eed19109",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture --no-display\n",
    "df_relations = pd.read_csv(data_folder+relation_file, sep=',', dtype='str')\n",
    "for c in df_relations:\n",
    "    df_relations[c] = df_relations[c].str.replace('\"','').str.replace('\\\\','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "579268ab-bebc-42d3-84c4-3892a424bafe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_relations = df_relations.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef71b8aa-1419-4174-922a-711ef70a3162",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture --no-display\n",
    "df_ids = pd.DataFrame(columns=[lbl_blank_node, lbl_node_id,'index', lbl_start_node , lbl_end_node ], dtype=str)\n",
    "for n in dfs:\n",
    "    df_ids = df_ids.append(dfs[n][[lbl_node_id ]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed21c399-a7cf-45db-b104-45c4598ee17b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture --no-display\n",
    "df_ids = df_ids.append(df_relations[['index',lbl_start_node , lbl_end_node ]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fab12b61-f168-4839-8813-708e676bf6c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ids=df_ids.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dc37cb8-392d-44dc-aee1-32216a88a688",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ids[lbl_blank_node ] = '_:x' + df_ids.index.astype(str) + ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57950544-6bb0-4b0d-b45b-c920ce0c227b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in dfs:\n",
    "    for c in dfs[n].columns:\n",
    "        dfs[n][c+'literal']  = [Literal(l) for l in dfs[n][c].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e04dee48-aeb3-45d7-8f2a-b98da41debef",
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in df_relations.columns:\n",
    "    df_relations[c+'literal']  = [Literal(l) for l in df_relations[c].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91c4d049-bacd-4447-b404-f6fac0dd5f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_dic={row[lbl_node_id]:row[lbl_blank_node] for idx, row in df_ids.iterrows()}\n",
    "node_dic.pop(np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5debea7-57da-4cbe-9471-99d4c79ac03a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_relations.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d6b595b-02aa-4160-b3ad-0eb6605fdfee",
   "metadata": {},
   "outputs": [],
   "source": [
    "edge_dic={(row['index'],row[lbl_start_node],row[lbl_end_node]):row[lbl_blank_node] for idx, row in df_ids.iterrows()}\n",
    "edge_dic.pop((np.nan,np.nan, np.nan))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ce7bf17-b20d-4e9b-9183-1a37b4ea9742",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(datetime.now()-start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f84bb265-026e-4c1a-8d51-b9cfbce218a4",
   "metadata": {},
   "source": [
    "# writing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57a0a090-7277-45fb-b888-d5cc1961ddd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture --no-display\n",
    "with open(output_file, 'w') as f:\n",
    "    f.write('@prefix ns1: <http://example.org/> .\\n')\n",
    "    f.write('@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\\n\\n\\n')\n",
    "    f.write('@prefix node: <http://nodeLabel.org/> > .\\n\\n\\n')\n",
    "    f.write('@prefix edge: <http://edgeLabel.org/> > .\\n\\n\\n')\n",
    "\n",
    "    for n in dfs:\n",
    "        print(n)\n",
    "\n",
    "        df = dfs[n].merge(df_ids, on=[lbl_node_id]).drop(columns=['index'])\n",
    "\n",
    "        for idx, row in df.iterrows():\n",
    "            f.write(row[lbl_blank_node] + ' ' + 'node:label' + ' ' + '\"' + Literal(n) + '\"' + ' .' + '\\n')\n",
    "            for c in df.columns:\n",
    "                if not (c.endswith('literal') or c in [lbl_blank_node ,lbl_start_node ,lbl_end_node , 'inedx']):\n",
    "                    if not str(row[c+'literal'])=='nan':\n",
    "                        #print(row[c+'literal'])\n",
    "                        f.write(row[lbl_blank_node]+ ' ' +URIRef('ns1:'+str(c))+' '+'\"' +str(row[c+'literal'])+'\"'+ ' .' +'\\n')\n",
    "\n",
    "    for idx, row in df_relations.iterrows():\n",
    "        \n",
    "        index = row['index'] \n",
    "        source_node=str(node_dic[row[lbl_start_node ]])\n",
    "        destination_node=str(node_dic[row[lbl_end_node ]])\n",
    "        relation=str(edge_dic[(index,row[lbl_start_node ],row[lbl_end_node ])])\n",
    "        predicate='edge:label'\n",
    "        rdfobject=str(Literal(row[lbl_relation_type+'literal']))\n",
    "        \n",
    "        if not result2RDFStar:\n",
    "            f.write(source_node+' '+URIRef('ns1:'+'InRelationTo')+' '+relation+' .'+'\\n')\n",
    "            f.write(relation+' '+URIRef('ns1:'+'PointsTo')+' '+destination_node+' .'+'\\n')\n",
    "            f.write(relation+' '+predicate+' '+'\"' +rdfobject+'\"'+' .'+'\\n')\n",
    "            \n",
    "        if result2RDFStar:\n",
    "            f.write(source_node + ' ' +URIRef('ns1:'+row[lbl_relation_type]) + ' '+ destination_node+ ' ' + '.' + '\\n' ) \n",
    " \n",
    "        for c in df_relations.columns:\n",
    "            if not(c.endswith('literal') or c in [lbl_start_node , lbl_relation_type, lbl_end_node, lbl_index ]):\n",
    "                if not(str(row[c])=='nan'):\n",
    "                    if result2RDFStar:\n",
    "                        f.write('<<'+ source_node + ' ' +URIRef('ns1:'+row[lbl_relation_type])+ ' '+ destination_node +'>>' + URIRef('ns1:'+str(c))+ ' '+'\"'+str(row[c+'literal'])+'\" .' +'\\n')\n",
    "                    else:\n",
    "                        f.write(relation+' '+URIRef('ns1:'+str(c))+' '+'\"' +str(row[c+'literal'])+'\"'+ ' .' +'\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92042a35-439f-4233-a932-8f3b79989a83",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(datetime.now()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe98c7dd-a691-493c-9700-ca0da67ad45d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51b61439-6bb9-4d81-ac76-edb802278485",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}