personalize_temporal_holdout/personalize_sims_smell_tests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding similar items + HPO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook follows a similar structure to the temporal holdout one, but uses aws-sims instead of HRNN. We look at the similar items found, and see if HPO helps improve things."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['AWS_DEFAULT_REGION']=\"us-east-1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix = str(np.random.uniform())[4:9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = \"demo-sims-\"+   suffix        # replace with the name of your S3 bucket\n",
    "filename = \"DEMO-sims.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "make_bucket: demo-sims-41593\r\n"
     ]
    }
   ],
   "source": [
    "!aws s3 mb s3://{bucket}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 46104  100 46104    0     0  1250k      0 --:--:-- --:--:-- --:--:-- 1286k\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  3320  100  3320    0     0  83000      0 --:--:-- --:--:-- --:--:-- 83000\n"
     ]
    }
   ],
   "source": [
    "!curl -O https://s3-us-west-2.amazonaws.com/personalize-cli-json-models/personalize.json\n",
    "!curl -O https://s3-us-west-2.amazonaws.com/personalize-cli-json-models/personalize-runtime.json\n",
    "!aws configure add-model --service-model file://`pwd`/personalize.json --service-name personalize\n",
    "!aws configure add-model --service-model file://`pwd`/personalize-runtime.json --service-name personalize-runtime\n",
    "\n",
    "personalize = boto3.client(service_name='personalize', endpoint_url='https://personalize.us-east-1.amazonaws.com')\n",
    "personalize_runtime = boto3.client(service_name='personalize-runtime', endpoint_url='https://personalize-runtime.us-east-1.amazonaws.com')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download and process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 5778k  100 5778k    0     0  8243k      0 --:--:-- --:--:-- --:--:-- 8243k\n",
      "Archive:  ml-1m.zip\n",
      "  inflating: ml-1m/movies.dat        \n",
      "  inflating: ml-1m/ratings.dat       \n",
      "  inflating: ml-1m/README            \n",
      "  inflating: ml-1m/users.dat         \n"
     ]
    }
   ],
   "source": [
    "!curl -O http://files.grouplens.org/datasets/movielens/ml-1m.zip\n",
    "!unzip -o ml-1m.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USER_ID</th>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th>RATING</th>\n",
       "      <th>TIMESTAMP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000207</th>\n",
       "      <td>6040</td>\n",
       "      <td>1096</td>\n",
       "      <td>4</td>\n",
       "      <td>956715648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000208</th>\n",
       "      <td>6040</td>\n",
       "      <td>1097</td>\n",
       "      <td>4</td>\n",
       "      <td>956715569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000209 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         USER_ID  ITEM_ID  RATING  TIMESTAMP\n",
       "0              1     1193       5  978300760\n",
       "1              1      661       3  978302109\n",
       "...          ...      ...     ...        ...\n",
       "1000207     6040     1096       4  956715648\n",
       "1000208     6040     1097       4  956715569\n",
       "\n",
       "[1000209 rows x 4 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./ml-1m/ratings.dat', sep='::', names=['USER_ID','ITEM_ID','RATING','TIMESTAMP'])\n",
    "pd.set_option('display.max_rows', 5)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique users 6040; unique items 3706\n"
     ]
    }
   ],
   "source": [
    "# data = data[data['RATING'] > 3.6]  # Use all data to predict view recommendations\n",
    "data = data[['USER_ID', 'ITEM_ID', 'TIMESTAMP']] # select columns that match the columns in the schema below\n",
    "print('unique users %d; unique items %d'%(\n",
    "    len(data['USER_ID'].unique()), len(data['ITEM_ID'].unique())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USER_ID,ITEM_ID,TIMESTAMP\r\n",
      "1,1193,978300760\r\n",
      "1,661,978302109\r\n",
      "1,914,978301968\r\n",
      "1,3408,978300275\r\n",
      "1,2355,978824291\r\n",
      "1,1197,978302268\r\n",
      "1,1287,978302039\r\n",
      "1,2804,978300719\r\n",
      "1,594,978302268\r\n"
     ]
    }
   ],
   "source": [
    "data.to_csv(filename, index=False)\n",
    "!head {filename}\n",
    "\n",
    "boto3.Session().resource('s3').Bucket(bucket).Object(filename).upload_file(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"schemaArn\": \"arn:aws:personalize:us-east-1:261294318658:schema/DEMO-sims-schema-41593\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"af9a8275-23d6-445a-afa6-98bad364a812\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 02:42:21 GMT\",\n",
      "      \"x-amzn-requestid\": \"af9a8275-23d6-445a-afa6-98bad364a812\",\n",
      "      \"content-length\": \"88\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "schema = {\n",
    "    \"type\": \"record\",\n",
    "    \"name\": \"Interactions\",\n",
    "    \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    "    \"fields\": [\n",
    "        {\n",
    "            \"name\": \"USER_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"ITEM_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"TIMESTAMP\",\n",
    "            \"type\": \"long\"\n",
    "        }\n",
    "    ],\n",
    "    \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_schema_response = personalize.create_schema(\n",
    "    name = \"DEMO-sims-schema-\"+suffix,\n",
    "    schema = json.dumps(schema)\n",
    ")\n",
    "\n",
    "schema_arn = create_schema_response['schemaArn']\n",
    "print(json.dumps(create_schema_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and Wait for Dataset Group creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetGroupArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-group/DEMO-sims-dataset-group-41593\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"1bb461dc-0cf8-42e5-b9aa-31e71743e548\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 02:42:27 GMT\",\n",
      "      \"x-amzn-requestid\": \"1bb461dc-0cf8-42e5-b9aa-31e71743e548\",\n",
      "      \"content-length\": \"108\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_dataset_group_response = personalize.create_dataset_group(\n",
    "    name = \"DEMO-sims-dataset-group-\"+suffix\n",
    ")\n",
    "\n",
    "dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n",
    "print(json.dumps(create_dataset_group_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetGroup: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_group_response = personalize.describe_dataset_group(\n",
    "        datasetGroupArn = dataset_group_arn\n",
    "    )\n",
    "    status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n",
    "    print(\"DatasetGroup: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset/DEMO-sims-dataset-group-41593/INTERACTIONS\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"e4b79afb-d339-42af-a404-4df350c3d23f\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 02:42:36 GMT\",\n",
      "      \"x-amzn-requestid\": \"e4b79afb-d339-42af-a404-4df350c3d23f\",\n",
      "      \"content-length\": \"110\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_type = \"INTERACTIONS\"\n",
    "create_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = schema_arn,\n",
    "    name = \"DEMO-sims-dataset-\"+suffix\n",
    ")\n",
    "\n",
    "dataset_arn = create_dataset_response['datasetArn']\n",
    "print(json.dumps(create_dataset_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make sure the bucket has the right permissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.client(\"s3\")\n",
    "\n",
    "policy = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Principal\": {\n",
    "                \"Service\": \"personalize.amazonaws.com\"\n",
    "            },\n",
    "            \"Action\": [\n",
    "                \"s3:GetObject\",\n",
    "                \"s3:ListBucket\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                \"arn:aws:s3:::{}\".format(bucket),\n",
    "                \"arn:aws:s3:::{}/*\".format(bucket)\n",
    "            ]\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create S3 Read-Only Access Role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from botocore.exceptions import ClientError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:iam::261294318658:role/PersonalizeS3Role-41593\n"
     ]
    }
   ],
   "source": [
    "iam = boto3.client(\"iam\")\n",
    "\n",
    "role_name = \"PersonalizeS3Role-\"+suffix\n",
    "assume_role_policy_document = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "          \"Effect\": \"Allow\",\n",
    "          \"Principal\": {\n",
    "            \"Service\": \"personalize.amazonaws.com\"\n",
    "          },\n",
    "          \"Action\": \"sts:AssumeRole\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "try:\n",
    "    create_role_response = iam.create_role(\n",
    "        RoleName = role_name,\n",
    "        AssumeRolePolicyDocument = json.dumps(assume_role_policy_document)\n",
    "    );\n",
    "\n",
    "    iam.attach_role_policy(\n",
    "        RoleName = role_name,\n",
    "        PolicyArn = \"arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess\"\n",
    "    );\n",
    "\n",
    "    role_arn = create_role_response[\"Role\"][\"Arn\"]\n",
    "except ClientError as e:\n",
    "    if e.response['Error']['Code'] == 'EntityAlreadyExists':\n",
    "        role_arn = iam.get_role(RoleName=role_name)['Role']['Arn']\n",
    "    else:\n",
    "        raise\n",
    "print(role_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare, Create, and Wait for Dataset Import Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetImportJobArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-import-job/DEMO-sims-dataset-import-job-41593\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"a8cd2d04-4b91-4205-8df9-b715c52e5a2e\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 02:42:58 GMT\",\n",
      "      \"x-amzn-requestid\": \"a8cd2d04-4b91-4205-8df9-b715c52e5a2e\",\n",
      "      \"content-length\": \"122\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"DEMO-sims-dataset-import-job-\"+suffix,\n",
    "    datasetArn = dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, filename)\n",
    "    },\n",
    "    roleArn = role_arn\n",
    ")\n",
    "\n",
    "dataset_import_job_arn = create_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Dataset Import Job and Dataset Import Job Run to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = describe_dataset_import_job_response[\"datasetImportJob\"]\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:personalize:::recipe/aws-hrnn\n",
      "arn:aws:personalize:::recipe/aws-hrnn-coldstart\n",
      "arn:aws:personalize:::recipe/aws-hrnn-metadata\n",
      "arn:aws:personalize:::recipe/aws-personalized-ranking\n",
      "arn:aws:personalize:::recipe/aws-popularity-count\n",
      "arn:aws:personalize:::recipe/aws-sims\n"
     ]
    }
   ],
   "source": [
    "recipe_list = personalize.list_recipes()\n",
    "for recipe in recipe_list['recipes']:\n",
    "    print(recipe['recipeArn'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are many recipes for different scenarios. In this example, we only have interactions data, so we will choose one from the basic recipes.\n",
    "\n",
    "| Feasible? | Recipe | Description \n",
    "|-------- | -------- |:------------\n",
    "| Y | aws-popularity-count | Calculates popularity of items based on count of events against that item in user-item interactions dataset.\n",
    "| Y | aws-hrnn | Predicts items a user will interact with. A hierarchical recurrent neural network which can model the temporal order of user-item interactions.\n",
    "| N - requires meta data | aws-hrnn-metadata | Predicts items a user will interact with. HRNN with additional features derived from contextual (user-item interaction metadata), user medata (user dataset) and item metadata (item dataset)\n",
    "| N - for bandits and requires meta data | aws-hrnn-coldstart | Predicts items a user will interact with. HRNN-metadata with with personalized exploration of new items.\n",
    "| N - for item-based queries | aws-sims | Computes items similar to a given item based on co-occurrence of item in same user history in user-item interaction dataset\n",
    "| N - for reranking a short list | aws-personalized-ranking | Reranks a list of items for a user. Trains on user-item interactions dataset. \n",
    "\n",
    "\n",
    "We (or autoML) can run all of these basic recipes and choose the best-performing model from internal metrics. We recommend comparisons, especially with popularity-baseline, to see the lifts in metrics via personalization. However, in this demo, we will pick one recipe - aws-sims, to illustrate smell tests."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe_arn = \"arn:aws:personalize:::recipe/aws-sims\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-sims-solution-41593\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"86338585-ef1f-403d-927a-0bf07c4e50a4\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 03:12:27 GMT\",\n",
      "      \"x-amzn-requestid\": \"86338585-ef1f-403d-927a-0bf07c4e50a4\",\n",
      "      \"content-length\": \"94\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_response = personalize.create_solution(\n",
    "    name = \"DEMO-sims-solution-\"+suffix,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    recipeArn = recipe_arn,\n",
    ")\n",
    "\n",
    "solution_arn = create_solution_response['solutionArn']\n",
    "print(json.dumps(create_solution_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-sims-solution-41593/5548c91f\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"be363337-237b-4056-9a23-8a0cb0df3a94\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 03:12:29 GMT\",\n",
      "      \"x-amzn-requestid\": \"be363337-237b-4056-9a23-8a0cb0df3a94\",\n",
      "      \"content-length\": \"110\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_version_response = personalize.create_solution_version(\n",
    "    solutionArn = solution_arn\n",
    ")\n",
    "\n",
    "solution_version_arn = create_solution_version_response['solutionVersionArn']\n",
    "print(json.dumps(create_solution_version_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Solution Version to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_solution_version_response = personalize.describe_solution_version(\n",
    "        solutionVersionArn = solution_version_arn\n",
    "    )\n",
    "    status = describe_solution_version_response[\"solutionVersion\"][\"status\"]\n",
    "    print(\"SolutionVersion: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Metrics of Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-sims-solution-41593/5548c91f\",\n",
      "  \"metrics\": {\n",
      "    \"coverage\": 0.4893,\n",
      "    \"mean_reciprocal_rank_at_25\": 0.0037,\n",
      "    \"normalized_discounted_cumulative_gain_at_10\": 0.0051,\n",
      "    \"normalized_discounted_cumulative_gain_at_25\": 0.0107,\n",
      "    \"normalized_discounted_cumulative_gain_at_5\": 0.0034,\n",
      "    \"precision_at_10\": 0.001,\n",
      "    \"precision_at_25\": 0.0013,\n",
      "    \"precision_at_5\": 0.001\n",
      "  },\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"20edf0f0-6ea2-41e1-b63e-30a9c5468960\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 04:05:36 GMT\",\n",
      "      \"x-amzn-requestid\": \"20edf0f0-6ea2-41e1-b63e-30a9c5468960\",\n",
      "      \"content-length\": \"406\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "get_metrics_response = personalize.get_solution_metrics(\n",
    "    solutionVersionArn = solution_version_arn\n",
    ")\n",
    "\n",
    "print(json.dumps(get_metrics_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create and Wait for Campaign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"campaignArn\": \"arn:aws:personalize:us-east-1:261294318658:campaign/DEMO-sims-campaign-41593\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"b1b79a7c-196d-4ca7-8c6c-8e029f7d266a\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 04:05:50 GMT\",\n",
      "      \"x-amzn-requestid\": \"b1b79a7c-196d-4ca7-8c6c-8e029f7d266a\",\n",
      "      \"content-length\": \"94\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_campaign_response = personalize.create_campaign(\n",
    "    name = \"DEMO-sims-campaign-\"+suffix,\n",
    "    solutionVersionArn = solution_version_arn,\n",
    "    minProvisionedTPS = 2,    \n",
    ")\n",
    "\n",
    "campaign_arn = create_campaign_response['campaignArn']\n",
    "print(json.dumps(create_campaign_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for Campaign to Have ACTIVE Status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Campaign: CREATE PENDING\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_campaign_response = personalize.describe_campaign(\n",
    "        campaignArn = campaign_arn\n",
    "    )\n",
    "    status = describe_campaign_response[\"campaign\"][\"status\"]\n",
    "    print(\"Campaign: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## to aid in interpretation, lets look at some items "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "movies = pd.read_csv('./ml-1m/movies.dat', sep='::', names=['ITEM_ID','title','genre'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies=movies.set_index('ITEM_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>genre</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      title                         genre\n",
       "ITEM_ID                                                                  \n",
       "1                          Toy Story (1995)   Animation|Children's|Comedy\n",
       "2                            Jumanji (1995)  Adventure|Children's|Fantasy\n",
       "3                   Grumpier Old Men (1995)                Comedy|Romance\n",
       "4                  Waiting to Exhale (1995)                  Comedy|Drama\n",
       "5        Father of the Bride Part II (1995)                        Comedy"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we pick a couple of items and look at if items found are generally of similar genres"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note, the model did not use this meta-data (genre) for training, this is a sanity or smell test to see if the model discovered similar items that 'make sense'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        itemId = str(5)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>genre</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>Gordy (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2350</th>\n",
       "      <td>Heart Condition (1990)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3313</th>\n",
       "      <td>Class Reunion (1982)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>Thin Line Between Love and Hate, A (1996)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>Meet the Deedles (1998)</td>\n",
       "      <td>Children's|Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             title              genre\n",
       "ITEM_ID                                                              \n",
       "243                                   Gordy (1995)             Comedy\n",
       "2350                        Heart Condition (1990)             Comedy\n",
       "3313                          Class Reunion (1982)             Comedy\n",
       "626      Thin Line Between Love and Hate, A (1996)             Comedy\n",
       "1822                       Meet the Deedles (1998)  Children's|Comedy"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[rec_items[:5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        itemId = str(2)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>genre</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Kids of the Round Table (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2079</th>\n",
       "      <td>Kidnapped (1960)</td>\n",
       "      <td>Children's|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1520</th>\n",
       "      <td>Commandments (1997)</td>\n",
       "      <td>Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>Amazing Panda Adventure, The (1995)</td>\n",
       "      <td>Adventure|Children's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>Thin Line Between Love and Hate, A (1996)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             title  \\\n",
       "ITEM_ID                                              \n",
       "56                  Kids of the Round Table (1995)   \n",
       "2079                              Kidnapped (1960)   \n",
       "1520                           Commandments (1997)   \n",
       "146            Amazing Panda Adventure, The (1995)   \n",
       "626      Thin Line Between Love and Hate, A (1996)   \n",
       "\n",
       "                                genre  \n",
       "ITEM_ID                                \n",
       "56       Adventure|Children's|Fantasy  \n",
       "2079                 Children's|Drama  \n",
       "1520                          Romance  \n",
       "146              Adventure|Children's  \n",
       "626                            Comedy  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[rec_items[:5]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do HPO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now see if doing HPO improves things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-sims-solution-hpo-41593\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"7aebbfcc-83f7-497f-8d1a-e7ff4b1d5a38\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 04:54:32 GMT\",\n",
      "      \"x-amzn-requestid\": \"7aebbfcc-83f7-497f-8d1a-e7ff4b1d5a38\",\n",
      "      \"content-length\": \"98\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_response = personalize.create_solution(\n",
    "    name = \"DEMO-sims-solution-hpo-\"+suffix,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    recipeArn = recipe_arn,\n",
    "    performHPO = True,\n",
    "    solutionConfig={\n",
    "        'hpoConfig': {\n",
    "            'hpoResourceConfig': {\n",
    "                  'maxNumberOfTrainingJobs': '40',\n",
    "                  'maxParallelTrainingJobs': '10'\n",
    "              }\n",
    "        }\n",
    "    }\n",
    ")\n",
    "\n",
    "solution_arn = create_solution_response['solutionArn']\n",
    "print(json.dumps(create_solution_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-sims-solution-hpo-41593/18b522a8\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"37bd77a1-cf9c-4bb6-8e37-540c89c653f8\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 04:54:52 GMT\",\n",
      "      \"x-amzn-requestid\": \"37bd77a1-cf9c-4bb6-8e37-540c89c653f8\",\n",
      "      \"content-length\": \"114\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_version_response = personalize.create_solution_version(\n",
    "    solutionArn = solution_arn\n",
    ")\n",
    "\n",
    "solution_version_arn = create_solution_version_response['solutionVersionArn']\n",
    "print(json.dumps(create_solution_version_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_solution_version_response = personalize.describe_solution_version(\n",
    "        solutionVersionArn = solution_version_arn\n",
    "    )\n",
    "    status = describe_solution_version_response[\"solutionVersion\"][\"status\"]\n",
    "    print(\"SolutionVersion: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-sims-solution-hpo-41593/18b522a8\",\n",
      "  \"metrics\": {\n",
      "    \"coverage\": 0.4929,\n",
      "    \"mean_reciprocal_rank_at_25\": 0.004,\n",
      "    \"normalized_discounted_cumulative_gain_at_10\": 0.0057,\n",
      "    \"normalized_discounted_cumulative_gain_at_25\": 0.0116,\n",
      "    \"normalized_discounted_cumulative_gain_at_5\": 0.004,\n",
      "    \"precision_at_10\": 0.001,\n",
      "    \"precision_at_25\": 0.0014,\n",
      "    \"precision_at_5\": 0.001\n",
      "  },\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"cd2365ff-eb40-4497-a822-041880435d4a\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 05:51:33 GMT\",\n",
      "      \"x-amzn-requestid\": \"cd2365ff-eb40-4497-a822-041880435d4a\",\n",
      "      \"content-length\": \"408\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "get_metrics_response = personalize.get_solution_metrics(\n",
    "    solutionVersionArn = solution_version_arn\n",
    ")\n",
    "\n",
    "print(json.dumps(get_metrics_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>genre</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Animation|Children's|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      title                         genre\n",
       "ITEM_ID                                                                  \n",
       "1                          Toy Story (1995)   Animation|Children's|Comedy\n",
       "2                            Jumanji (1995)  Adventure|Children's|Fantasy\n",
       "3                   Grumpier Old Men (1995)                Comedy|Romance\n",
       "4                  Waiting to Exhale (1995)                  Comedy|Drama\n",
       "5        Father of the Bride Part II (1995)                        Comedy"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>genre</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>Gordy (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2350</th>\n",
       "      <td>Heart Condition (1990)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3313</th>\n",
       "      <td>Class Reunion (1982)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>Thin Line Between Love and Hate, A (1996)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1822</th>\n",
       "      <td>Meet the Deedles (1998)</td>\n",
       "      <td>Children's|Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             title              genre\n",
       "ITEM_ID                                                              \n",
       "243                                   Gordy (1995)             Comedy\n",
       "2350                        Heart Condition (1990)             Comedy\n",
       "3313                          Class Reunion (1982)             Comedy\n",
       "626      Thin Line Between Love and Hate, A (1996)             Comedy\n",
       "1822                       Meet the Deedles (1998)  Children's|Comedy"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        itemId = str(5)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "movies.loc[rec_items[:5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>genre</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Kids of the Round Table (1995)</td>\n",
       "      <td>Adventure|Children's|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2079</th>\n",
       "      <td>Kidnapped (1960)</td>\n",
       "      <td>Children's|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1520</th>\n",
       "      <td>Commandments (1997)</td>\n",
       "      <td>Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>Amazing Panda Adventure, The (1995)</td>\n",
       "      <td>Adventure|Children's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>626</th>\n",
       "      <td>Thin Line Between Love and Hate, A (1996)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             title  \\\n",
       "ITEM_ID                                              \n",
       "56                  Kids of the Round Table (1995)   \n",
       "2079                              Kidnapped (1960)   \n",
       "1520                           Commandments (1997)   \n",
       "146            Amazing Panda Adventure, The (1995)   \n",
       "626      Thin Line Between Love and Hate, A (1996)   \n",
       "\n",
       "                                genre  \n",
       "ITEM_ID                                \n",
       "56       Adventure|Children's|Fantasy  \n",
       "2079                 Children's|Drama  \n",
       "1520                          Romance  \n",
       "146              Adventure|Children's  \n",
       "626                            Comedy  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        itemId = str(2)\n",
    "    )\n",
    "rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "movies.loc[rec_items[:5]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that items returned by sims after HPO are 'more similar'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}