CubeFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CubeFlow: Money Laundering Detection with Coupled Tensors\n",
    "CubeFlow is a scalable, flow-based approach to spot fraud from a mass of transactions by modeling them as two coupled tensors and applying a novel multi-attribute metric which can reveal the transfer chains accurately."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remember to add spartan to you PATH\n",
    "import sys\n",
    "sys.path.append(\"/<dir to spartan2>/spartan2\")\n",
    "import spartan as st"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set parameters\n",
    "alpha: the coefficientof imbalance cost rate in the range of 0 to 1\n",
    "\n",
    "k: find top k dense blocks\n",
    "\n",
    "dim: dimensions of input data (support 3 or 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.8\n",
    "k = 1\n",
    "dim = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Path of input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xy_path = f'./inputData/CFD-{dim}/fs1.csv'\n",
    "zy_path = f'./inputData/CFD-{dim}/fs2.csv'\n",
    "gt_path = f'./inputData/CFD-{dim}/gt.npy'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Path to save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outpath = '' # '': not save results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load example data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "amt_tensor = st.loadTensor(path=xy_path, header=None)\n",
    "cmt_tensor = st.loadTensor(path=zy_path, header=None)\n",
    "\n",
    "amt_stensor = amt_tensor.toSTensor(hasvalue=True)\n",
    "cmt_stensor = cmt_tensor.toSTensor(hasvalue=True)\n",
    "print(amt_stensor.shape)\n",
    "print(cmt_stensor.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run as a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cf = st.CubeFlow([amt_stensor, cmt_stensor], alpha=alpha, k=k, dim=dim, outpath=outpath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Run the algorithm and obtain two dense blocks as return value:**\n",
    "\n",
    "$maxsize$ is the block size limit.\n",
    "\n",
    "$maxsize$ can be an integer (-1 or positive), and $maxsize==-1$ means no size limit.\n",
    "\n",
    "$maxsize$ can be a tuple which contains the node size limit for each dimension of the block, e.g.,(5,10,5). Similarly, each element of $maxsize$ should be an integer (-1 or positive).\n",
    "\n",
    "The results is a list of top-k suspicious blocks. Each block constains [[detected nodes in each partite], score]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = cf.run(del_type=1, maxsize=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation\n",
    "Functions for calculating scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_f1(detectSet, trueSet, am_tensor, cm_tensor, has_t=False):\n",
    "    # TP\n",
    "    set_a = detectSet[0] & trueSet[0]\n",
    "    set_m = detectSet[1] & trueSet[1]\n",
    "    set_c = detectSet[2] & trueSet[2]\n",
    "    fs1 = am_tensor.data\n",
    "    fs2 = cm_tensor.data\n",
    "    if has_t:\n",
    "        set_t = detectSet[3] & trueSet[3]\n",
    "\n",
    "        tp1 = fs1[(fs1[0].isin(set_a)) & (fs1[1].isin(set_m)) & (fs[2].isin(set_t))]\n",
    "        tp2 = fs2[(fs2[1].isin(set_m)) & (fs2[0].isin(set_c)) & (fs[2].isin(set_t))]\n",
    "        # FP+TP\n",
    "        fptp1 = fs1[(fs1[0].isin(detectSet[0])) & (fs1[1].isin(detectSet[1])) & (fs1[2].isin(detectSet[3]))]\n",
    "        fptp2 = fs2[(fs2[1].isin(detectSet[1])) & (fs2[0].isin(detectSet[2])) & (fs2[2].isin(detectSet[3]))]\n",
    "        # FN+TP\n",
    "        fntp1 = fs1[(fs1[0].isin(trueSet[0])) & (fs1[1].isin(trueSet[1])) & (fs1[2].isin(trueSet[3]))]\n",
    "        fntp2 = fs2[(fs2[1].isin(trueSet[1])) & (fs2[0].isin(trueSet[2])) & (fs2[2].isin(trueSet[3]))]\n",
    "\n",
    "    else:\n",
    "        tp1 = fs1[(fs1[0].isin(set_a)) & (fs1[1].isin(set_m))]\n",
    "        tp2 = fs2[(fs2[1].isin(set_m)) & (fs2[0].isin(set_c))]\n",
    "        # FP+TP\n",
    "        fptp1 = fs1[(fs1[0].isin(detectSet[0])) & (fs1[1].isin(detectSet[1]))]\n",
    "        fptp2 = fs2[(fs2[1].isin(detectSet[1])) & (fs2[0].isin(detectSet[2]))]\n",
    "        # FN+TP\n",
    "        fntp1 = fs1[(fs1[0].isin(trueSet[0])) & (fs1[1].isin(trueSet[1]))]\n",
    "        fntp2 = fs2[(fs2[1].isin(trueSet[1])) & (fs2[0].isin(trueSet[2]))]\n",
    "        \n",
    "    tpm = tp1[3].sum() + tp2[3].sum()\n",
    "    tpfpm = fptp1[3].sum() + fptp2[3].sum()\n",
    "    tpfnm = fntp1[3].sum() + fntp2[3].sum()\n",
    "    # precision\n",
    "    precision = tpm / tpfpm\n",
    "    print(f'precision:{precision}')\n",
    "    # recall\n",
    "    recall = tpm / tpfnm\n",
    "    print(f'recall:{recall}')\n",
    "    # F1\n",
    "    f1_score = 0.0\n",
    "    if (precision + recall) > 0:\n",
    "        f1_score = 2*precision*recall/(precision+recall)\n",
    "    print(f'F1 score:{f1_score}')\n",
    "    return f1_score, precision, recall\n",
    "\n",
    "def get_groundtruth(path):\n",
    "    gt = np.load(path, allow_pickle=True)\n",
    "    gt = gt.tolist()\n",
    "    return gt\n",
    "\n",
    "def find_top_k_res(real_res, k):\n",
    "    top_k_res = []\n",
    "    top_k_a = set()\n",
    "    top_k_m = set()\n",
    "    top_k_c = set()\n",
    "    for i in range(k):\n",
    "        top_k_a = top_k_a.union(real_res[i][0][0])\n",
    "        top_k_m = top_k_m.union(real_res[i][0][1])\n",
    "        top_k_c = top_k_c.union(real_res[i][0][2])\n",
    "    top_k_res.append(top_k_a)\n",
    "    top_k_res.append(top_k_m)\n",
    "    top_k_res.append(top_k_c)\n",
    "    return top_k_res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Report F-measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "gt = get_groundtruth(gt_path)\n",
    "for i in range(1,k+1):\n",
    "    print(f'Top {i} block:')\n",
    "    top_k_res = find_top_k_res(res, i)\n",
    "    f1, precision, recall = cal_f1(top_k_res, gt, amt_tensor,cmt_tensor)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}