05-counties-download-graphs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download counties' street networks\n",
    "\n",
    "Using census tigerline shapefile of 2017 counties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8.1\n",
      "2.1\n"
     ]
    }
   ],
   "source": [
    "import config\n",
    "import geopandas as gpd\n",
    "import json\n",
    "import logging as lg\n",
    "import networkx as nx\n",
    "import os\n",
    "import osmnx as ox\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "ox.config(use_cache=True,\n",
    "          log_file=True,\n",
    "          log_console=True,\n",
    "          log_filename='download-counties',\n",
    "          cache_folder=config.counties_cache_folder)\n",
    "\n",
    "print(ox.__version__)\n",
    "print(nx.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "network_type = 'drive'\n",
    "retain_all = True\n",
    "simplify = True\n",
    "\n",
    "input_folder = 'input_data/counties' #tiger place shapefiles\n",
    "output_folder_shapefile = config.counties_shapefile_folder #where to save graph shapefiles\n",
    "output_folder_graphml = config.counties_graphml_folder #where to save graphml files\n",
    "output_folder_lists = config.counties_lists_folder #where to save node/edge lists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('input_data/states_by_fips.json') as f:\n",
    "    fips_to_state = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3233"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counties = gpd.read_file(input_folder).sort_values(by='ALAND', ascending=True)\n",
    "len(counties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'init': 'epsg:4269'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'init': 'epsg:4326'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tigerline data is epsg:4269, but osm uses epsg:4326, so project it\n",
    "print(counties.crs)\n",
    "counties = counties.to_crs({'init':'epsg:4326'})\n",
    "counties.crs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ALAND</th>\n",
       "      <th>NAMELSAD</th>\n",
       "      <th>GEOID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1327</th>\n",
       "      <td>88635832004</td>\n",
       "      <td>Valdez-Cordova Census Area</td>\n",
       "      <td>02261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>92326483031</td>\n",
       "      <td>Northwest Arctic Borough</td>\n",
       "      <td>02188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1509</th>\n",
       "      <td>105228855314</td>\n",
       "      <td>Bethel Census Area</td>\n",
       "      <td>02050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2246</th>\n",
       "      <td>230056014591</td>\n",
       "      <td>North Slope Borough</td>\n",
       "      <td>02185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>521</th>\n",
       "      <td>377030936019</td>\n",
       "      <td>Yukon-Koyukuk Census Area</td>\n",
       "      <td>02290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ALAND                    NAMELSAD  GEOID\n",
       "1327   88635832004  Valdez-Cordova Census Area  02261\n",
       "243    92326483031    Northwest Arctic Borough  02188\n",
       "1509  105228855314          Bethel Census Area  02050\n",
       "2246  230056014591         North Slope Borough  02185\n",
       "521   377030936019   Yukon-Koyukuk Census Area  02290"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counties[['ALAND', 'NAMELSAD', 'GEOID']].tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the aleutian islands county because it crosses the 180th meridian\n",
    "counties = counties[counties['GEOID'] != '02016']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3141"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def make_fips_state(fips):\n",
    "    if fips in fips_to_state:\n",
    "        return '{}_{}'.format(fips, fips_to_state[fips]['abbreviation'])\n",
    "\n",
    "counties['fips_state'] = counties['STATEFP'].map(make_fips_state)\n",
    "counties = counties.dropna(subset=['fips_state'])\n",
    "len(counties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_geoid_name(row):\n",
    "    return '{}_{}'.format(row['GEOID'], row['NAMELSAD']).replace(',', '').replace(' ', '_').replace('/', '_')\n",
    "\n",
    "counties['geoid_name'] = counties.apply(make_geoid_name, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fips_state': '02_AK',\n",
       " 'geoid_name': '02290_Yukon-Koyukuk_Census_Area',\n",
       " 'polygon': <shapely.geometry.polygon.Polygon at 0x2b41de28128>}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "queries = counties.apply(lambda row: {'fips_state':row['fips_state'],\n",
    "                                      'geoid_name':row['geoid_name'],\n",
    "                                      'polygon':row['geometry']}, axis=1).tolist()\n",
    "queries = list(reversed(queries))\n",
    "queries[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and save graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_node_edge_lists(G, lists_folder):\n",
    "    \n",
    "    # save node and edge lists as csv\n",
    "    nodes, edges = ox.graph_to_gdfs(G, node_geometry=False, fill_edge_geometry=False)\n",
    "    edges['length'] = edges['length'].round(3)\n",
    "\n",
    "    ecols = ['u', 'v', 'key', 'oneway', 'highway', 'name', 'length',\n",
    "             'lanes', 'width', 'est_width', 'maxspeed', 'access', 'service',\n",
    "             'bridge', 'tunnel', 'area', 'junction', 'osmid', 'ref']\n",
    "\n",
    "    edges = edges.drop(columns=['geometry']).reindex(columns=ecols)\n",
    "    nodes = nodes.reindex(columns=['osmid', 'x', 'y', 'ref', 'highway'])\n",
    "\n",
    "    if not os.path.exists(lists_folder):\n",
    "        os.makedirs(lists_folder)\n",
    "    nodes.to_csv('{}/node_list.csv'.format(lists_folder), index=False, encoding='utf-8')\n",
    "    edges.to_csv('{}/edge_list.csv'.format(lists_folder), index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished making graphs in 6.4 seconds\n"
     ]
    }
   ],
   "source": [
    "all_start_time = time.time()\n",
    "\n",
    "for query in queries:\n",
    "    try:\n",
    "        shapefile_folder = '{}/{}'.format(output_folder_shapefile, query['fips_state'])\n",
    "        shapefile_file = query['geoid_name']\n",
    "        lists_folder = '{}/{}/{}'.format(output_folder_lists, query['fips_state'], query['geoid_name'])\n",
    "        graphml_folder = '{}/{}'.format(output_folder_graphml, query['fips_state'])\n",
    "        graphml_file = '{}.graphml'.format(query['geoid_name'])\n",
    "        graphml_path = '{}/{}'.format(graphml_folder, graphml_file)\n",
    "\n",
    "        if not os.path.exists(graphml_path):\n",
    "            \n",
    "            G = ox.graph_from_polygon(polygon=query['polygon'],\n",
    "                                      network_type=network_type,\n",
    "                                      name=query['geoid_name'],\n",
    "                                      retain_all=retain_all,\n",
    "                                      simplify=simplify)\n",
    "            \n",
    "            save_node_edge_lists(G, lists_folder)\n",
    "            ox.save_graphml(G, folder=graphml_folder, filename=graphml_file)\n",
    "            ox.save_graph_shapefile(G, folder=shapefile_folder, filename=shapefile_file)\n",
    "    \n",
    "    except Exception as e:\n",
    "        ox.log('\"{}\" failed: {}'.format(query['geoid_name'], e), level=lg.ERROR)\n",
    "        \n",
    "print('Finished making graphs in {:,.1f} seconds'.format(time.time() - all_start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ? seconds last time "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}