playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from import_file import *\n",
    "from helpers.parallel_helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49483865960020695"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.power(99/100,70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# from __future__ import division\n",
    "from import_file import *\n",
    "from helpers.parallel_helper import *\n",
    "load_libs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "0.90,0.962,0.954,0.908,0.975,0.941,0.905,0.916,\n",
    "0.85,0.742,0.819,0.884,0.901,0.868,0.793,0.727\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "speed_params = k, h, scale, loc = 0.351, 0.626, 5.095, 2.508\n",
    "\n",
    "vonmises_params = [\n",
    "[1.101, 1.430, 0.115],\n",
    "[16.989, 1.155, 0.239],\n",
    "[14.661, 4.446, 0.301],\n",
    "[2.793, 0.733, 0.172],\n",
    "[2.512, 4.282, 0.173]]\n",
    "\n",
    "connection_params = [\n",
    "[0.746, 0.774, 0.486],\n",
    "[0.883, 4.073, 0.514]]\n",
    "\n",
    "al_params = [speed_params, vonmises_params, connection_params]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BIN_WIDTH=1\n",
    "PLOT_AXIS_RANGE = arange(-12, 12+BIN_WIDTH, BIN_WIDTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = Y = PLOT_AXIS_RANGE\n",
    "XX, YY=np.meshgrid(X,Y)\n",
    "coords=np.array((XX.ravel(), YY.ravel())).T\n",
    "\n",
    "x = [ np.sqrt(x**2 + y**2) for x,y in coords]\n",
    "alpha = [ np.arctan2(y, x) for x,y in coords]\n",
    "al_pdf = angular_linear_pdf(x, alpha, speed_params, vonmises_params, connection_params, cartesian=True)\n",
    "al_Z = al_pdf.reshape(XX.shape)\n",
    "\n",
    "fig = plt.figure(figsize=(3.5,2.5))\n",
    "plot_2d_prob_density(X,Y,al_Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "original_dim = int(np.sqrt(len(al_pdf)))\n",
    "al_pdf_ = al_pdf.reshape(original_dim, original_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ssss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats import vonmises\n",
    "kappa1, loc1, w1 = 5, 1, 2000\n",
    "kappa2, loc2, w2 = 2, -2, 5000\n",
    "x = np.linspace(0, 2*pi, 50)\n",
    "plot(x, vonmises.pdf(x, kappa1, loc=loc1)*w1+vonmises.pdf(x, kappa2, loc=loc2)*w2, label='vonmises pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "samples_1 = vonmises.rvs(kappa1, loc=loc1, size=2000)\n",
    "samples_2 = vonmises.rvs(kappa2, loc=loc2, size=5000)\n",
    "samples = np.append(samples_1,samples_2)\n",
    "pd.DataFrame(samples).hist(bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = [cos(samples), sin(samples)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%R -i data -o result,theta,alpha\n",
    "result = movMF(data, 2, nruns = 10)\n",
    "theta = result$theta\n",
    "alpha = result$alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kappas = [sqrt(sum(power(theta_,2))) for theta_  in theta]\n",
    "locs = [np.arctan2(theta_[1], theta_[0]) for theta_  in theta]\n",
    "print(kappas, locs, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = np.linspace(0, 2*pi, 50)\n",
    "y = 0\n",
    "for i in range(2):\n",
    "    y = y + vonmises.pdf(x, kappas[i], loc=locs[i])*alpha[i]\n",
    "plot(x, y, label='vonmises pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats import kappa4, vonmises\n",
    "\n",
    "speed_params = k, h, scale, loc = 0.351, 0.626, 5.095, 2.508\n",
    "\n",
    "vonmises_params = [\n",
    "[1.101, 1.430, 0.115],\n",
    "[16.989, 1.155, 0.239],\n",
    "[14.661, 4.446, 0.301],\n",
    "[2.793, 0.733, 0.172],\n",
    "[2.512, 4.282, 0.173]]\n",
    "\n",
    "connection_params = [\n",
    "[0.746, 0.774, 0.486],\n",
    "[0.883, 4.073, 0.514]]\n",
    "\n",
    "angular_linear_params = [speed_params, vonmises_params, connection_params]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def angular_linear_pdf(x, alpha, speed_params, vonmises_params, connection_params):\n",
    "#     # 1. Speed\n",
    "#     k, h, scale, loc = speed_params\n",
    "#     x_pdf = kappa4.pdf(x, h, k, loc=loc, scale=scale)\n",
    "#     x_cdf = kappa4.cdf(x, h, k, loc=loc, scale=scale)\n",
    "#     # 2. Direction\n",
    "#     alpha_pdf, alpha_cdf = 0, 0\n",
    "#     for k, u, w in vonmises_params:\n",
    "#         alpha_pdf = alpha_pdf + vonmises.pdf(alpha, k, loc=u, scale=1)*w\n",
    "#         alpha_cdf = alpha_cdf + vonmises.cdf(alpha, k, loc=u, scale=1)*w\n",
    "#     # 3. Connection\n",
    "#     phi = 2*pi*(x_cdf-alpha_cdf)\n",
    "#     phi_pdf = np.sum([vonmises.pdf(phi, k, loc=u, scale=1)*w for k, u, w in connection_params])\n",
    "#     return 2*pi*x_pdf*alpha_pdf*phi_pdf\n",
    "# def f_al(x, alpha):\n",
    "#     return angular_linear_pdf(x, alpha, speed_params, vonmises_params, connection_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "angle_radian, incre_radian = radians(150), radians(10)\n",
    "incre=10\n",
    "bins = arange(0, 20, 1)\n",
    "bin_width=0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "density_expected_al_ = [sp.integrate.nquad(f_al, [[x_, x_+bin_width], [angle_radian-incre_radian/2, angle_radian+incre_radian/2]])\n",
    "                             for x_ in bins[:-1]]\n",
    "density_expected_al = array(list(zip(*density_expected_al_))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "densities = Parallel(n_jobs=-1)(delayed(f_al_parallel)(angular_linear_params, angle_radian, incre_radian, bins, bin_width) \n",
    "                                        for angle in arange(0, 360, incre))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot(x, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot(x/pi*180, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import lmoments3 as lm\n",
    "from lmoments3 import distr\n",
    "data = [2.0, 3.0, 4.0, 2.4, 5.5, 1.2, 5.4, 2.2, 7.1, 1.3, 1.5]\n",
    "paras = distr.gam.lmom_fit(data)\n",
    "paras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fitted_gam = distr.gam(**paras)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fitted_gam.pdf(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "abs(np.log10(1-0.999))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "abs(np.log10(1-0.9936))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "1- power(10,-2.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Higher -> better\n",
    "abs(np.log10(1-0.99929))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.log10(1-0.92255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set_matplotlib_formats('png', 'svg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from nbconvert import HTMLExporter\n",
    "# import codecs\n",
    "# import nbformat\n",
    "# exporter = HTMLExporter()\n",
    "# output_notebook = nbformat.read(read_file, as_version=4)\n",
    "# output, resources = exporter.from_notebook_node(output_notebook)\n",
    "# codecs.open(output_file, 'w', encoding='utf-8').write(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ssss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Title"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Title 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Title 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in arange(0.4,1,0.1):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame([[1,2,3], [2,3,4]], index=[2,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_path='10_metre_surface.csv'\n",
    "df = pd.read_csv(file_path, header=0, skipinitialspace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['speed']=np.sqrt(df.U**2+df.V**2)\n",
    "df['dir']=np.degrees(np.arctan2(df.V, df.U))%360\n",
    "df['time']=pd.to_datetime('1979-01-01T00:00:00Z')+pd.to_timedelta(df['time'], unit='h')\n",
    "df['date']=df['time'].dt.strftime('%Y%m%d')\n",
    "df['HrMn']=df['time'].dt.strftime('%H%m')\n",
    "df.set_index(['time'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['HrMn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['2010':'2014']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bins = arange(0, 20,0.5)\n",
    "df['speed'].hist(bins=bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bins = arange(0, 361)\n",
    "df['dir'].hist(bins=bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "figs = [try_plot(y) for y in arange(3)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "figs = Parallel(n_jobs=-1)(delayed(try_plot)(y) for y in arange(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "figs = Parallel(n_jobs=2)(delayed(try_plot)(y) for y in arange(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nbformat\n",
    "from nbconvert.preprocessors import ExecutePreprocessor\n",
    "\n",
    "with open('test.ipynb') as f:\n",
    "    nb = nbformat.read(f, as_version=4)\n",
    "    \n",
    "ep = ExecutePreprocessor(timeout=600, kernel_name='python3')\n",
    "\n",
    "ep.preprocess(nb, {'metadata': {'path': './'}})\n",
    "\n",
    "with open('executed_notebook.ipynb', 'wt') as f:\n",
    "    nbformat.write(nb, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a, b = np.radians([2, 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "person1 = {'type': 1, 'name': 'Jhon', 'surname': 'Smith', 'phone': '555-1234'}\n",
    "person2 = {'type': 2, 'name': 'Jannette', 'surname': 'Jhonson', 'phone': 1000000.00}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame([person1, person2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ssss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "file_dir = \"./data/NCDC/europe/spain/gran_canaria/\"\n",
    "for file in os.listdir(file_dir):\n",
    "    if file.endswith(\"stn.txt\"):\n",
    "        stn_file_path = file_dir+file\n",
    "        print(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(stn_file_path) as fp:\n",
    "    for i, line in enumerate(fp):\n",
    "        if i == 2:\n",
    "            lat, long= line.split()[-3:-1]\n",
    "            \n",
    "lat, long = float(lat), float(long)\n",
    "print(lat, long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import folium\n",
    "map_osm = folium.Map(location=[lat, long], zoom_start=4)\n",
    "folium.Marker([lat, long]).add_to(map_osm)\n",
    "map_osm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ssss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "file_path = 'D:/NDAWN/table.csv'\n",
    "df = pd.read_csv(file_path, skipinitialspace=True, dtype={'Year':'object', 'Month':'object', \n",
    "                                                          'Day':'object'})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Output format: date, HrMn, dir, speed, speed_max\n",
    "df['HrMn']=df['Hour'].apply(lambda x: \"%04d\"%x ) \n",
    "df['date']=df['Year']+df['Month']+df['Day']\n",
    "df = df[['date','HrMn','Avg Wind Speed', 'Max Wind Speed', 'Avg Wind Dir', 'Avg Wind Dir SD']]\n",
    "df.rename(columns={'Avg Wind Speed': 'speed', 'Max Wind Speed': 'speed_max', \n",
    "                   'Avg Wind Dir': 'dir', 'Avg Wind Dir SD': 'dir_std'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "\n",
    "def my_to_datetime(date_str):\n",
    "    if date_str[8:10] != '24':\n",
    "        return pd.to_datetime(date_str, format='%Y%m%d%H%M')\n",
    "\n",
    "    date_str = date_str[0:8] + '00' + date_str[10:]\n",
    "    return pd.to_datetime(date_str, format='%Y%m%d%H%M') + \\\n",
    "           dt.timedelta(days=1)\n",
    "   \n",
    "df['time']=df[\"date\"].astype(str).map(str)+df[\"HrMn\"]\n",
    "df['time']=df['time'].apply(my_to_datetime)\n",
    "df['date']=df['time'].apply(lambda x: x.strftime('%Y%m%d'))\n",
    "df['HrMn']=df['time'].apply(lambda x: x.strftime('%H%M'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bins=arange(0,df['speed'].max())\n",
    "df['speed'].hist(bins=bins, normed=True,alpha=0.5,label='min')\n",
    "plt_configure(legend=True)\n",
    "\n",
    "plt.figure()\n",
    "bins=arange(0,362)\n",
    "df['dir'].hist(bins=bins, normed=True,alpha=0.5,label='min')\n",
    "plt_configure(legend=True, figsize=(20,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('D:/NDAWN/baker_hr_avg.csv', index=False, float_format='%.2f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gmm = [[0.670,-6.409,-4.276,9.773,10.219,-0.200],\n",
    "[0.202,2.084,9.018,6.664,6.802,-0.108],\n",
    "[0.128,9.989,-12.887,7.075,6.579,-0.311],]\n",
    "speed_unit_text = ' (knot)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig_gmm, ax = plt.subplots(figsize=(3.5,3.5))\n",
    "plot_gmm_ellipses(gmm, ax=ax, xlabel='x'+speed_unit_text, ylabel='y'+speed_unit_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = [1,2,3,4]\n",
    "y = [2,3,4,5]\n",
    "plot(x, y )\n",
    "plt_configure(xlabel=\"ln($V$)\", ylabel=\"ln $P$\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.random.rand(50, 4), columns=['a', 'b', 'c', 'd'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mpl.rcParams['patch.force_edgecolor'] = True\n",
    "df.plot.scatter(x='a', y='b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# mpl.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame(np.random.rand(10, 4), columns=['a', 'b', 'c', 'd'])\n",
    "\n",
    "df2.plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "x = [1,2,3,4,5]\n",
    "y = [5,2,6,7,2]\n",
    "ax.set(xlabel='X Label', ylabel='Y Label', title='My Title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = [1,2,3,4,5]\n",
    "y = [5,2,6,7,2]\n",
    "plot(x, y, '-', label='test')\n",
    "plt_configure(title = 'This is my title', figsize=(4,4), xlabel='X name', ylabel='Y name', legend=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fig, ax = plt.subplots()\n",
    "x = [1,2,3,4,5]\n",
    "y = [5,2,6,7,2]\n",
    "plot(x, y, '-', label='test')\n",
    "ax = plt.gca()\n",
    "ax.set_title('This is my title')\n",
    "ax.set_xlabel('X name')\n",
    "ax.set_ylabel('Y name')\n",
    "ax.legend()\n",
    "plt.gcf().set_size_inches(4,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mpl.rcParams['axes.prop_cycle'].by_key()['color']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mpl.rcParams['axes.prop_cycle']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prop_cycle = mpl.rcParams['axes.prop_cycle']\n",
    "# prop_cycle = iter(mpl.rcParams['axes.color_cycle'])\n",
    "x = [1,2,3,4,5]\n",
    "y = [2,5,1,2,1]\n",
    "y_menas = [3,3,3,3,3]\n",
    "plot(x, y, '-', color=next(prop_cycle))\n",
    "plot(x, y_menas, '--', color=next(prop_cycle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from cycler import cycler\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x = np.linspace(0, 2 * np.pi)\n",
    "offsets = np.linspace(0, 2*np.pi, 4, endpoint=False)\n",
    "# Create array with shifted-sine curve along each column\n",
    "yy = np.transpose([np.sin(x + phi) for phi in offsets])\n",
    "\n",
    "# 1. Setting prop cycle on default rc parameter\n",
    "plt.rc('lines', linewidth=4)\n",
    "plt.rc('axes', prop_cycle=(cycler('color', ['r', 'g', 'b', 'y']) +\n",
    "                           cycler('linestyle', ['-', '--', ':', '-.'])))\n",
    "fig, (ax0, ax1) = plt.subplots(nrows=2)\n",
    "ax0.plot(yy)\n",
    "ax0.set_title('Set default color cycle to rgby')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(np.linspace(0,30))\n",
    "\n",
    "a, b, c = test = 25, 1,3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('test.txt', skipinitialspace=True, sep=' ', error_bad_lines=False,\n",
    "                names = ['stationID','time','vis','day_type','dir','speed','dir_max','speed_max','unname','visual_range', 'unknown'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['dir'].apply(lambda x: str(x).isdigit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "s = pd.Series(['0.55', '2', -3, '1234t1s1'])\n",
    "pd.to_numeric(s, downcast='integer',errors='coerce' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('DSM/minute_avg.csv', header=0, skipinitialspace=True, dtype={'HrMn': object})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['year'] = df['date'].apply(lambda x: str(x)[:6])\n",
    "df['year'].value_counts().sort_index().plot(kind='bar', figsize=(20,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "display(df[df.duplicated('date')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gmm_em_result = [[0.670,-6.409,-4.276,9.773,10.219,-0.200],\n",
    "[0.202,2.084,9.018,6.664,6.80,-0.108],\n",
    "[0.128,9.989,-12.887,7.075,6.579,-0.311]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_gmm_ellipses(gmm_em_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x=[1,2,3,4,5]\n",
    "y=[4,5,5,2,1]\n",
    "plot(x,y,'-')\n",
    "plt.gca().set_xlim([1, 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file_path = './data/NCDC/uk/tiree/dat.txt'\n",
    "df = pd.read_csv(file_path,header=0, skipinitialspace=True)\n",
    "df.rename(columns={'Date':'date','Dir':'dir','Spd':'speed','Type':'type','I.1':'wind_type'}, inplace=True)\n",
    "df = df[['date','HrMn','type','dir','speed','wind_type' ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = df.query(\"(dir <= 999) & (speed < 100) & \\\n",
    "              (date >= 20000000) & (date < 20160000) \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df = pd.read_csv('DSM/DSM.csv',header=0, skipinitialspace=True, dtype={'HrMn': object})\n",
    "# df['speed']=pd.to_numeric(df['speed'], downcast='integer', errors='coerce')\n",
    "# df['dir']=pd.to_numeric(df['dir'], downcast='integer', errors='coerce')\n",
    "# df['dir_max']=pd.to_numeric(df['dir_max'], downcast='integer', errors='coerce')\n",
    "# df['speed_max']=pd.to_numeric(df['speed_max'], downcast='integer', errors='coerce')\n",
    "# df = df.dropna()\n",
    "# df=df[['date','HrMn','dir','speed','dir_max','speed_max']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df['Hr']=df['HrMn'].apply(lambda x: int(str(x)[:-2]))\n",
    "# df.dir = np.radians(df.dir)\n",
    "# temp_df =df.groupby(['date', 'Hr']).agg({'speed' : np.mean,'dir': sp.stats.circmean,'speed_max': np.max}).reset_index()\n",
    "# df.dir = np.degrees(df.dir)\n",
    "# temp_df.dir = np.degrees(temp_df.dir)\n",
    "# temp_df['HrMn']=temp_df['Hr'].apply(lambda x: \"%02d00\"%x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_weibul(df_speed):\n",
    "    max_speed = df_speed.max()\n",
    "    x = linspace(0, max_speed)\n",
    "    data_size = len(df_speed)\n",
    "    \n",
    "    df_speed.hist(bins=arange(0, max_speed), alpha=0.5, label='Data', normed=True)  \n",
    "\n",
    "    weibull_params = sp.stats.exponweib.fit(df_speed, floc=0, f0=1)\n",
    "    y_weibull = sp.stats.exponweib.pdf(x, *weibull_params)\n",
    "    plot(x, y_weibull, '--', color='black',label='Weibull') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_df = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for time_limit in [0, 20050000, 20100000]:\n",
    "    plt.figure()\n",
    "    df_speed=temp_df.query('date > @time_limit')['speed']\n",
    "    fit_weibul(df_speed)\n",
    "align_figures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_df2, speed_redistribution_info = randomize_speed(temp_df, 'up')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fit_weibul(temp_df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_2016 = df.query('date >= 20150000 & date <= 20160000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fit_weibul(df_2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots()\n",
    "\n",
    "ecdf = sm.distributions.ECDF(df.speed)\n",
    "x = linspace(0, df.speed.max())\n",
    "y = ecdf(x)\n",
    "plot(log(x), log(-log(1-y)),'o', label='ECDF')\n",
    "\n",
    "_, k_shape, _, lamb_scale = sp.stats.exponweib.fit(df.speed, floc=0, f0=1)\n",
    "y=1-exp(-(x/lamb_scale)**k_shape) # cdf\n",
    "plot(log(x), log(-log(1-y)),'-', label='Weibull')\n",
    "plt_configure(xlabel=\"ln(V)\", ylabel=\"ln(-ln(1-P)\",legend={'loc':'best'},figsize=(4,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ssss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ssss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(sns.color_palette(\"hls\",10 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(sns.color_palette(\"hls\",6 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(sns.color_palette(\"hls\",4 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(sns.color_palette(\"hls\",3 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(mpl.rcParams['axes.color_cycle'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(sns.color_palette(\"Blues_r\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.palplot(sns.cubehelix_palette(reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "from numpy.random import randn\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "\n",
    "\n",
    "def to_percent(y, position):\n",
    "    # Ignore the passed in position. This has the effect of scaling the default\n",
    "    # tick locations.\n",
    "    s = str(100 * y)\n",
    "\n",
    "    # The percent symbol needs escaping in latex\n",
    "    if matplotlib.rcParams['text.usetex'] is True:\n",
    "        return s + r'$\\%$'\n",
    "    else:\n",
    "        return s + '%'\n",
    "\n",
    "x = randn(5000)\n",
    "\n",
    "# Make a normed histogram. It'll be multiplied by 100 later.\n",
    "plt.hist(x, bins=50, normed=True, edgecolor = \"black\")\n",
    "\n",
    "# Create the formatter using the function to_percent. This multiplies all the\n",
    "# default labels by 100, making them all percentages\n",
    "formatter = FuncFormatter(to_percent)\n",
    "\n",
    "# Set the formatter\n",
    "plt.gca().yaxis.set_major_formatter(formatter)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "# Loading the Digits dataset\n",
    "digits = datasets.load_digits()\n",
    "\n",
    "# To apply an classifier on this data, we need to flatten the image, to\n",
    "# turn the data in a (samples, feature) matrix:\n",
    "n_samples = len(digits.images)\n",
    "X = digits.images.reshape((n_samples, -1))\n",
    "y = digits.target\n",
    "\n",
    "# Split the dataset in two equal parts\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.5, random_state=0)\n",
    "\n",
    "# Set the parameters by cross-validation\n",
    "tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],\n",
    "                     'C': [1, 10, 100, 1000]},\n",
    "                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]\n",
    "\n",
    "scores = ['precision', 'recall']\n",
    "\n",
    "for score in scores:\n",
    "    print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "    print()\n",
    "\n",
    "    clf = GridSearchCV(SVC(C=1), tuned_parameters, cv=5,\n",
    "                       scoring='%s_macro' % score)\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "    print(\"Detailed classification report:\")\n",
    "    print()\n",
    "    print(\"The model is trained on the full development set.\")\n",
    "    print(\"The scores are computed on the full evaluation set.\")\n",
    "    print()\n",
    "    y_true, y_pred = y_test, clf.predict(X_test)\n",
    "    print(classification_report(y_true, y_pred))\n",
    "    print()\n",
    "\n",
    "# Note the problem is too easy: the hyperparameter plateau is too flat and the\n",
    "# output model is the same for precision and recall with ties in quality."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# from __future__ import division\n",
    "from import_file import *\n",
    "from helpers.parallel_helper import *\n",
    "load_libs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prop_cycle = mpl.rcParams['axes.prop_cycle']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prop_cycle"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}