examples/calibration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robot Calibration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nominal Robot\n",
    "- A nominal robot model:\n",
    "    - Represents what the robot manufacturer intended as a kinematic model\n",
    "    - Is mathematically ideal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pybotics.robot import Robot\n",
    "from pybotics.predefined_models import ur10\n",
    "\n",
    "nominal_robot = Robot.from_parameters(ur10())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alpha</th>\n",
       "      <th>a</th>\n",
       "      <th>theta</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>118.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.570796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.141593</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>612.7</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>571.6</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>163.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.570796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>115.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.570796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.141593</td>\n",
       "      <td>92.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      alpha      a     theta      d\n",
       "0  0.000000    0.0  0.000000  118.0\n",
       "1  1.570796    0.0  3.141593    0.0\n",
       "2  0.000000  612.7  0.000000    0.0\n",
       "3  0.000000  571.6  0.000000  163.9\n",
       "4 -1.570796    0.0  0.000000  115.7\n",
       "5  1.570796    0.0  3.141593   92.2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def display_robot_kinematics(robot: Robot):\n",
    "    df = pd.DataFrame(robot.kinematic_chain.matrix)\n",
    "    df.columns = [\"alpha\", \"a\", \"theta\", \"d\"]\n",
    "    display(df)\n",
    "\n",
    "display_robot_kinematics(nominal_robot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Real* Robots\n",
    "- *Real* robots do not conform perfectly to the nominal parameters\n",
    "- Small errors in the robot model can generate large errors in Cartesian position\n",
    "- Sources of errors include, but are not limited to:\n",
    "    - Kinematic errors\n",
    "        - Mechanical tolerances\n",
    "        - Angle offsets\n",
    "    - Non-kinematic errors\n",
    "        - Joint stiffness\n",
    "        - Gravity\n",
    "        - Temperature\n",
    "        - Friction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>alpha</th>\n",
       "      <th>a</th>\n",
       "      <th>theta</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.001602</td>\n",
       "      <td>118.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.570796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.140136</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>612.7</td>\n",
       "      <td>0.000937</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>571.6</td>\n",
       "      <td>0.001712</td>\n",
       "      <td>163.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.570796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001548</td>\n",
       "      <td>115.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.570796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.141530</td>\n",
       "      <td>92.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      alpha      a     theta      d\n",
       "0  0.000000    0.0 -0.001602  118.0\n",
       "1  1.570796    0.0  3.140136    0.0\n",
       "2  0.000000  612.7  0.000937    0.0\n",
       "3  0.000000  571.6  0.001712  163.9\n",
       "4 -1.570796    0.0  0.001548  115.7\n",
       "5  1.570796    0.0  3.141530   92.2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "\n",
    "real_robot = deepcopy(nominal_robot)\n",
    "\n",
    "# let's pretend our real robot has small joint offsets\n",
    "# in real life, this would be a joint mastering issue (level-1 calibration)\n",
    "# https://en.wikipedia.org/wiki/Robot_calibration\n",
    "for link in real_robot.kinematic_chain.links:\n",
    "    link.theta += np.random.uniform(\n",
    "        low=np.deg2rad(-0.1),\n",
    "        high=np.deg2rad(0.1)\n",
    "    )\n",
    "\n",
    "display_robot_kinematics(real_robot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get *Real* (aka Measured) Poses\n",
    "- In real life, these poses would be measured using metrology equipment (e.g., laser tracker, CMM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "joints = []\n",
    "positions = []\n",
    "for i in range(1000):\n",
    "    q = real_robot.random_joints()\n",
    "    pose = real_robot.fk(q)\n",
    "    \n",
    "    joints.append(q)\n",
    "    positions.append(pose[:-1,-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.024511</td>\n",
       "      <td>-0.090671</td>\n",
       "      <td>0.024519</td>\n",
       "      <td>0.013319</td>\n",
       "      <td>0.057824</td>\n",
       "      <td>0.002825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.845469</td>\n",
       "      <td>1.810471</td>\n",
       "      <td>1.833426</td>\n",
       "      <td>1.839980</td>\n",
       "      <td>1.809561</td>\n",
       "      <td>1.826789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.120738</td>\n",
       "      <td>-3.136021</td>\n",
       "      <td>-3.140768</td>\n",
       "      <td>-3.138327</td>\n",
       "      <td>-3.133358</td>\n",
       "      <td>-3.141046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-1.644452</td>\n",
       "      <td>-1.625226</td>\n",
       "      <td>-1.578201</td>\n",
       "      <td>-1.574109</td>\n",
       "      <td>-1.508009</td>\n",
       "      <td>-1.603747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.075271</td>\n",
       "      <td>-0.169728</td>\n",
       "      <td>-0.115088</td>\n",
       "      <td>0.011472</td>\n",
       "      <td>0.119836</td>\n",
       "      <td>0.036910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.610011</td>\n",
       "      <td>1.517423</td>\n",
       "      <td>1.630127</td>\n",
       "      <td>1.642295</td>\n",
       "      <td>1.594479</td>\n",
       "      <td>1.579081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.135209</td>\n",
       "      <td>3.136369</td>\n",
       "      <td>3.123256</td>\n",
       "      <td>3.137543</td>\n",
       "      <td>3.135128</td>\n",
       "      <td>3.138303</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3            4  \\\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000  1000.000000   \n",
       "mean     -0.024511    -0.090671     0.024519     0.013319     0.057824   \n",
       "std       1.845469     1.810471     1.833426     1.839980     1.809561   \n",
       "min      -3.120738    -3.136021    -3.140768    -3.138327    -3.133358   \n",
       "25%      -1.644452    -1.625226    -1.578201    -1.574109    -1.508009   \n",
       "50%      -0.075271    -0.169728    -0.115088     0.011472     0.119836   \n",
       "75%       1.610011     1.517423     1.630127     1.642295     1.594479   \n",
       "max       3.135209     3.136369     3.123256     3.137543     3.135128   \n",
       "\n",
       "                 5  \n",
       "count  1000.000000  \n",
       "mean      0.002825  \n",
       "std       1.826789  \n",
       "min      -3.141046  \n",
       "25%      -1.603747  \n",
       "50%       0.036910  \n",
       "75%       1.579081  \n",
       "max       3.138303  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(joints).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-1.557721</td>\n",
       "      <td>16.865645</td>\n",
       "      <td>157.122190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>431.902411</td>\n",
       "      <td>443.597388</td>\n",
       "      <td>593.908420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1301.872708</td>\n",
       "      <td>-1305.844242</td>\n",
       "      <td>-1170.407200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-239.188266</td>\n",
       "      <td>-222.494618</td>\n",
       "      <td>-263.998275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-15.584041</td>\n",
       "      <td>15.233551</td>\n",
       "      <td>151.033202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>265.114264</td>\n",
       "      <td>274.892061</td>\n",
       "      <td>616.140406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1267.182128</td>\n",
       "      <td>1260.598491</td>\n",
       "      <td>1404.081554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 x            y            z\n",
       "count  1000.000000  1000.000000  1000.000000\n",
       "mean     -1.557721    16.865645   157.122190\n",
       "std     431.902411   443.597388   593.908420\n",
       "min   -1301.872708 -1305.844242 -1170.407200\n",
       "25%    -239.188266  -222.494618  -263.998275\n",
       "50%     -15.584041    15.233551   151.033202\n",
       "75%     265.114264   274.892061   616.140406\n",
       "max    1267.182128  1260.598491  1404.081554"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(positions, columns=['x','y','z']).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split Calibration and Validation Measures\n",
    "- A portion of the measured configurations and positions should be set aside for validation after calibration (i.e., optimization)\n",
    "    - This is to prevent/check the optimized model for overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "split = train_test_split(joints, positions, test_size=0.3)\n",
    "\n",
    "train_joints = split[0]\n",
    "test_joints = split[1]\n",
    "\n",
    "train_positions = split[2]\n",
    "test_positions = split[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Nominal Position Errors\n",
    "- These nominal model is our starting point for calibration\n",
    "- The errors are in millimetres "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    300.000000\n",
       "mean       1.261451\n",
       "std        0.511057\n",
       "min        0.331676\n",
       "25%        0.871703\n",
       "50%        1.143849\n",
       "75%        1.693312\n",
       "max        2.428211\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pybotics.optimization import compute_absolute_errors\n",
    "\n",
    "nominal_errors = compute_absolute_errors(\n",
    "    qs=test_joints,\n",
    "    positions=test_positions,\n",
    "    robot=nominal_robot\n",
    ")\n",
    "\n",
    "display(pd.Series(nominal_errors).describe())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calibration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False,  True, False],\n",
       "       [False, False,  True, False],\n",
       "       [False, False,  True, False],\n",
       "       [False, False,  True, False],\n",
       "       [False, False,  True, False],\n",
       "       [False, False,  True, False]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pybotics.optimization import OptimizationHandler\n",
    "\n",
    "# init calibration handler\n",
    "handler = OptimizationHandler(nominal_robot)\n",
    "\n",
    "# set handler to solve for theta parameters\n",
    "kc_mask_matrix = np.zeros_like(nominal_robot.kinematic_chain.matrix, dtype=bool)\n",
    "kc_mask_matrix[:,2] = True\n",
    "display(kc_mask_matrix)\n",
    "\n",
    "handler.kinematic_chain_mask = kc_mask_matrix.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Iteration     Total nfev        Cost      Cost reduction    Step norm     Optimality   \n",
      "       0              1         6.5073e+02                                    4.68e+05    \n",
      "       1              7         5.3649e+01      5.97e+02       4.34e-03       1.35e+05    \n",
      "       2              9         2.1031e-01      5.34e+01       2.17e-03       3.23e+03    \n",
      "       3             12         9.2586e-03      2.01e-01       2.71e-04       1.36e+03    \n",
      "       4             15         2.1788e-04      9.04e-03       3.39e-05       2.15e+02    \n",
      "       5             18         4.5223e-05      1.73e-04       4.24e-06       1.25e+02    \n",
      "       6             20         1.2495e-06      4.40e-05       2.12e-06       3.44e+00    \n",
      "       7             22         9.3713e-07      3.12e-07       1.06e-06       7.69e+00    \n",
      "       8             24         4.6008e-07      4.77e-07       2.65e-07       7.89e+00    \n",
      "       9             25         8.7686e-08      3.72e-07       2.65e-07       5.52e+00    \n",
      "      10             28         1.8195e-08      6.95e-08       3.31e-08       2.70e+00    \n",
      "`xtol` termination condition is satisfied.\n",
      "Function evaluations 28, initial cost 6.5073e+02, final cost 1.8195e-08, first-order optimality 2.70e+00.\n"
     ]
    }
   ],
   "source": [
    "from scipy.optimize import least_squares\n",
    "from pybotics.optimization import optimize_accuracy\n",
    "\n",
    "# run optimization\n",
    "result = least_squares(\n",
    "    fun=optimize_accuracy,\n",
    "    x0=handler.generate_optimization_vector(),\n",
    "    args=(handler, train_joints, train_positions),\n",
    "    verbose=2\n",
    ")  # type: scipy.optimize.OptimizeResult"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "- A calibrated robot model is never perfect in real life\n",
    "    - The goal is often to reduce the max error under a desired threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    300.000000\n",
       "mean       0.000007\n",
       "std        0.000003\n",
       "min        0.000001\n",
       "25%        0.000004\n",
       "50%        0.000007\n",
       "75%        0.000009\n",
       "max        0.000012\n",
       "dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "calibrated_robot = handler.robot\n",
    "calibrated_errors = compute_absolute_errors(\n",
    "    qs=test_joints,\n",
    "    positions=test_positions,\n",
    "    robot=calibrated_robot\n",
    ")\n",
    "\n",
    "display(pd.Series(calibrated_errors).describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAELCAYAAADeNe2OAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAcUklEQVR4nO3deZRU5b3u8e/DEFuJogxyTYg2KhFEBpvWQAj3gGiuFwxGMc4REpccXSbeeFwngifrije6rq5jxOmcJCRE0EOiBkVNHG4kgSRmmUiTKJNGHJqIAwJxQOLs7/5Ru9uix+qmdlVX7+ezVq3a+609PFU0v9r11q53KyIwM7Ps6FHuAGZmVlou/GZmGePCb2aWMS78ZmYZ48JvZpYxLvxmZhmTauGXtK+kpZKekvSkpPGS+kl6WNLG5H6/NDOYmdmulOZ5/JIWA7+PiB9L+gSwF3AZ8PeIuFrSHGC/iLi0re0MGDAgqqurU8tpZtYdrV69eltEDGzanlrhl9QXeBw4OPJ2IumvwKSIeFnSAcDKiDisrW3V1tZGXV1dKjnNzLorSasjorZpe5pdPUOArcAtkv4i6ceS+gCDIuLlZJlXgEEpZjAzsybSLPy9gBrg+xFxJLATmJO/QPJJoMWPHJJmS6qTVLd169YUY5qZZUuahX8zsDki/pTMLyX3RrAl6eIhuX+1pZUjYkFE1EZE7cCBzbqozMysk1Ir/BHxCvCCpIb++ynABuA+YGbSNhO4N60MZmbWXK+Ut/9NYElyRs9zwNfIvdncKelcYBNwasoZzMwsT6qFPyIeB5p9o0zu6N/MzMrAv9w1M8sYF34zs4xJu4/fzCxTqufcv8t8/dXTypSkdT7iNzPLGBd+M7OMceE3M8sYF34zs4xx4TczyxgXfjOzjHHhNzPLGBd+M7OMceE3M8sYF34zs4xx4TczyxgXfjOzjHHhNzPLGBd+M7OMceE3M8sYF34zs4xx4TczyxgX/q5mXt/czcwsJS78ZmYZ48JvZpYxLvxmZhnjwm9mljEu/GZmGePCb2aWMb3S3LikemAH8CHwQUTUSuoH3AFUA/XAqRHxWpo5zMzsY6U44p8cEWMiojaZnwP8OiKGAr9O5s3MrETK0dVzIrA4mV4MfLkMGczMMivVrh4ggF9JCuCHEbEAGBQRLyePvwIMamlFSbOB2QAHHnhgyjHNzDques795Y7QKWkX/i9ExIuS9gcelvRU/oMREcmbQjPJm8QCgNra2haXMTOzjku1qyciXkzuXwWWAUcDWyQdAJDcv5pmBjMz21VqhV9SH0l7N0wDXwTWAfcBM5PFZgL3ppXBzMyaS7OrZxCwTFLDfn4aEQ9JWgXcKelcYBNwaooZzMysidQKf0Q8B4xuoX07MCWt/ZqZWdv8y10zs4xx4TczyxgXfjOzjHHhNzPLGBd+M7OMceE3M8sYF34zs4xx4TczyxgXfjOzjHHhNzPLGBd+M7OMceE3M8sYF34zs4xx4TczyxgXfjOzjHHhNzPLGBd+M7OMceE3M8sYF34zs4xx4TczyxgXfjOzjHHhNzPLGBd+M7OM6VXuAAbM61vuBGaWIT7iNzPLGBd+M7OMceE3M8uY1Au/pJ6S/iLpl8n8EEl/kvSMpDskfSLtDGZm9rFSHPH/L+DJvPlrgPkRcSjwGnBuCTKYmVki1cIvaTAwDfhxMi/gGGBpsshi4MtpZjAzs12lfcR/PfBt4KNkvj/wekR8kMxvBj7d0oqSZkuqk1S3devWlGOamWVHaoVf0gnAqxGxujPrR8SCiKiNiNqBAwcWOZ2ZWXal+QOuCcB0SVOBKmAf4AZgX0m9kqP+wcCLKWYwM7MmUjvij4i5ETE4IqqB04HfRMRZwArglGSxmcC9aWUwM7PmynEe/6XAv0h6hlyf/8IyZDAzy6ySjNUTESuBlcn0c8DRpdivmZk151/umplljAu/mVnGuPCbmWWMC7+ZWcYUVPgljUw7iJmZlUahZ/X8p6Q9gEXAkoh4I71IFaS1K2fN88tjZl1XQUf8ETEROAv4DLBa0k8lHZdqMjMzS0XBffwRsRH4DrkfYP0TcKOkpySdnFY4MzMrvkL7+EdJmk9uXP1jgC9FxPBken6K+czMrMgK7eO/idyY+pdFxNsNjRHxkqTvpJLMzMxSUWjhnwa8HREfAkjqAVRFxD8i4rbU0pmZWdEV2se/HNgzb36vpM3MzCpMoUf8VRHxVsNMRLwlaa+UMmVDa6eCmllmVM+5H4D6q6eVdL+FHvHvlFTTMCNpLPB2G8ubmVkXVegR/7eAn0t6CRDw34DTUktlZmapKajwR8QqScOAw5Kmv0bE++nFMjOztHTkQixHAdXJOjWSiIhbU0lV6Vrqv/cwDmbWRRRU+CXdBhwCPA58mDQH4MJvZlZhCj3irwUOj4hIM4yZmaWv0LN61pH7QtfMzCpcoUf8A4ANkh4D3m1ojIjpqaQyM7PUFFr456UZwszMSqfQ0zl/K+kgYGhELE9+tdsz3WhmZpaGQodlPg9YCvwwafo0cE9aoczMLD2Ffrl7ITABeBMaL8qyf1qhzMwsPYUW/ncj4r2GGUm9yJ3Hb2ZmFabQwv9bSZcBeybX2v058Iu2VpBUJekxSU9IWi/piqR9iKQ/SXpG0h2SPrF7T8HMzDqi0MI/B9gKrAX+GXiA3PV32/IucExEjAbGAMdLGgdcA8yPiEOB14BzOxPczMw6p9Czej4CfpTcCpL8yrdhDP/eyS3IXaf3zKR9MblTRb9f6HbNzGz3FDpWz/O00KcfEQe3s15PYDVwKPAfwLPA6xHxQbLIZnJnCJmZWYl0ZKyeBlXAV4B+7a2UXKN3jKR9gWXAsEKDSZoNzAY48MADC12tuHyVLDPrhgrq44+I7Xm3FyPienIXYC9IRLwOrADGA/smZwUBDAZebGWdBRFRGxG1AwcOLHRXZmbWjkK7emryZnuQ+wTQ5rqSBgLvR8TrkvYEjiP3xe4K4BTgdmAmcG8ncpuZWScV2tXzvbzpD4B64NR21jkAWJz08/cA7oyIX0raANwu6UrgL8DCjkU2M7PdUehZPZM7uuGIWAMc2UL7c8DRHd2emZkVR6FdPf/S1uMRcV1x4piZWdo6clbPUcB9yfyXgMeAjWmEMjOz9BRa+AcDNRGxA0DSPOD+iDg7rWBmZl1B9Zz7S7aP+qsLPllytxQ6ZMMg4L28+feSNjMzqzCFHvHfCjwmaVky/2Vywy2YmVmFKfSsnqskPQhMTJq+FhF/SS+WmZmlpdAjfoC9gDcj4hZJAyUNiYjn0wrW7Xj4BzPrIgq99OLlwKXA3KSpN/BfaYUyM7P0FPrl7knAdGAnQES8BOydVigzM0tPoV0970VESAoASX1SzGSwa9fQvDfKl8PMSib/1NE0T+0s9Ij/Tkk/JDey5nnAcjpwURYzM+s6Cj2r59rkWrtvAocB/zsiHk41mZmZpaLdwp+Mrrk8GajNxd7MrMK1W/gj4kNJH0nqGxHubDYz64CmQz6UaliGthT65e5bwFpJD5Oc2QMQERelksrMzFJTaOG/O7mZmVmFa+/yiQdGxN8iwuPymJkVQSlG+2xPe6dz3tMwIemulLOYmVkJtFf4lTd9cJpBzMysNNor/NHKtJmZVaj2vtwdLelNckf+eybTJPMREfukms7MzIquzcIfET1LFcTMzEqj0LF6zMysm3DhNzPLGBd+M7OMceE3M8sYF34zs4zpyMXWO0TSZ4BbgUHkfgOwICJukNQPuAOoBuqBUyPitbRymJm1pCsMnVAuaR7xfwBcEhGHA+OACyUdDswBfh0RQ4FfJ/NmZlYiqRX+iHg5Iv6cTO8AngQ+DZwINAz6thj4cloZzMysuZL08UuqBo4E/gQMioiXk4deIdcVZGZmJZJ64Zf0SeAu4FsR8Wb+YxERtDIGkKTZkuok1W3dujXtmGZmmZFq4ZfUm1zRXxIRDRdy2SLpgOTxA4BXW1o3IhZERG1E1A4cODDNmGZmmZJa4ZckYCHwZERcl/fQfcDMZHomcG9aGczMrLnUTucEJgBfJXet3seTtsuAq4E7JZ0LbAJOTTGDmZk1kVrhj4hH2PVCLvmmpLVfMzNrm3+5a2aWMS78ZmYZ48JvZpYxLvxmZhnjwm9mljEu/GZmGePCb2aWMS78ZmYZ48JvZpYxLvxmZhnjwm9mljEu/GZmGePCb2aWMS78ZmYZ48JvZpYxLvxmZhnjwm9mljEu/GZmGePCb2aWMS78ZmYZ48JvZpYxLvxmZhnjwm9mljEu/GZmGePCb2aWMS78ZmYZ48JvZpYxqRV+ST+R9KqkdXlt/SQ9LGljcr9fWvvvVub1LXcCM+tGeqW47UXAzcCteW1zgF9HxNWS5iTzl6aYwSrM+++/z+bNm3nnnXfKHaXbqaqqYvDgwfTu3bvcUazMUiv8EfE7SdVNmk8EJiXTi4GVuPBbns2bN7P33ntTXV2NpHLH6TYigu3bt7N582aGDBlS7jhWZqXu4x8UES8n068Ag1pbUNJsSXWS6rZu3VqadFZ277zzDv3793fRLzJJ9O/f35+kDCjjl7sREUC08fiCiKiNiNqBAweWMJmVm4t+Ovy6WoNSF/4tkg4ASO5fLfH+zdoliUsuuaRx/tprr2XevHlF2fbUqVN5/fXXO7XuvHnzuPbaa4uSw7ItzS93W3IfMBO4Orm/t8T7twpTPef+om6v/upp7S6zxx57cPfddzN37lwGDBhQ1P0/8MADRd2eWWekeTrnz4BHgcMkbZZ0LrmCf5ykjcCxybxZl9KrVy9mz57N/Pnzmz1WX1/PMcccw6hRo5gyZQp/+9vfAJg1axYXXHAB48aN4+CDD2blypV8/etfZ/jw4cyaNatx/erqarZt20Z9fT3Dhw/nvPPOY8SIEXzxi1/k7bffBuBHP/oRRx11FKNHj2bGjBn84x//KMnztuxIrfBHxBkRcUBE9I6IwRGxMCK2R8SUiBgaEcdGxN/T2r/Z7rjwwgtZsmQJb7zxxi7t3/zmN5k5cyZr1qzhrLPO4qKLLmp87LXXXuPRRx9l/vz5TJ8+nYsvvpj169ezdu1aHn/88Wb72LhxIxdeeCHr169n33335a677gLg5JNPZtWqVTzxxBMMHz6chQsXpvtkLXP8y12zFuyzzz6cc8453Hjjjbu0P/roo5x55pkAfPWrX+WRRx5pfOxLX/oSkhg5ciSDBg1i5MiR9OjRgxEjRlBfX99sH0OGDGHMmDEAjB07tnGZdevWMXHiREaOHMmSJUtYv359Ok/SMsuF36wV3/rWt1i4cCE7d+4saPk99tgDgB49ejRON8x/8MEHrS4P0LNnz8ZlZs2axc0338zatWu5/PLLfQqmFZ0Lv1kr+vXrx6mnnrpLV8vnP/95br/9dgCWLFnCxIkTi77fHTt2cMABB/D++++zZMmSom/fzIXfrA2XXHIJ27Zta5y/6aabuOWWWxg1ahS33XYbN9xwQ9H3+d3vfpfPfe5zTJgwgWHDhhV9+2bK/Y6qa6utrY26urrS77grDY427432l+kGnnzySYYPH17uGN2WX9+PFftU4WIr5NTj9khaHRG1Tdt9xG9mljEu/GZmGePCb2aWMS78ZmYZ48JvZpYxLvxmZhnjwm/WgldeeYXTTz+dQw45hLFjxzJ16lSefvrpVpf/5Cc/CcBLL73EKaecAsCiRYv4xje+sVs5rr/++g4P0rZy5UpOOOGE3dqvdW+lHpa59LrSufjWccX+9yvg9xARwUknncTMmTMbf6X7xBNPsGXLFj772c+2ue6nPvUpli5dWnCciCAi6NGj5WOw66+/nrPPPpu99tqr4G1a91A95/6inMvfEh/xmzWxYsUKevfuzfnnn9/YNnr0aI488kimTJlCTU0NI0eO5N57m19Oor6+niOOOKJx/oUXXmDSpEkMHTqUK664onGZww47jHPOOYcjjjiCF154gQsuuIDa2lpGjBjB5ZdfDsCNN97ISy+9xOTJk5k8eTIAv/rVrxg/fjw1NTV85Stf4a233gLgoYceYtiwYdTU1HD33Xen9tpY99D9j/jNOmjdunWMHTu2WXtVVRXLli1jn332Ydu2bYwbN47p06e3eUnDxx57jHXr1rHXXntx1FFHMW3aNAYMGMDGjRtZvHgx48aNA+Cqq66iX79+fPjhh0yZMoU1a9Zw0UUXcd1117FixQoGDBjAtm3buPLKK1m+fDl9+vThmmuu4brrruPb3/425513Hr/5zW849NBDOe2001J7bax78BG/WYEigssuu4xRo0Zx7LHH8uKLL7Jly5Y21znuuOPo378/e+65JyeffHLjMM4HHXRQY9EHuPPOO6mpqeHII49k/fr1bNiwodm2/vjHP7JhwwYmTJjAmDFjWLx4MZs2beKpp55iyJAhDB06FEmcffbZxX3i1u34iN+siREjRrTYT79kyRK2bt3K6tWr6d27N9XV1e0Omdz000DDfJ8+fRrbnn/+ea699lpWrVrFfvvtx6xZs1rcbkRw3HHH8bOf/WyX9pYu8mLWFh/xmzVxzDHH8O6777JgwYLGtjVr1rBp0yb2339/evfuzYoVK9i0aVO723r44Yf5+9//zttvv80999zDhAkTmi3z5ptv0qdPH/r27cuWLVt48MEHGx/be++92bFjBwDjxo3jD3/4A8888wwAO3fu5Omnn2bYsGHU19fz7LPPAjR7YzBryoXfrAlJLFu2jOXLl3PIIYcwYsQI5s6dy9SpU6mrq2PkyJHceuutBQ2ZfPTRRzNjxgxGjRrFjBkzqK1tNlBi4xfHw4YN48wzz9zlzWH27Nkcf/zxTJ48mYEDB7Jo0SLOOOMMRo0axfjx43nqqaeoqqpiwYIFTJs2jZqaGvbff/+ivh7W/XT/YZm7y+mcHpbZisCv78e6+rDMsPtDM3tYZjMzA1z4zcwyx4XfzCxjXPity6mE750qkV9Xa+DCb11KVVUV27dvd5Eqsohg+/btVFVVlTuKdQH+AZd1KYMHD2bz5s1s3bq13FG6naqqKgYPHlzuGNYFlKXwSzoeuAHoCfw4Iq4uRw7renr37s2QIUPKHcOsWyt5V4+knsB/AP8TOBw4Q9Lhpc5hZpZV5ejjPxp4JiKei4j3gNuBE8uQw8wsk8pR+D8NvJA3vzlpMzOzEuiyX+5Kmg3MTmbfkvTXvIf7Am+0Md1wPwDY1skI+dvtyOMttbeWN3++pWU+zn9F62O+dyBbIcu0lq+l+famO/v6lzN/flta+Tv7t9N0vtL+9pvOV1r+zv3f7Zhd9qFr2l+mjfa+wEEtbqHh0m+lugHjgf+XNz8XmNvBbSxoazrvvm43ci7ozOMttbeWt6XMxcjfXvaO5G9rvoB/h4rL36Qtlfyd/dvpSP6u+Ldf6fkr/f9u/q0cXT2rgKGShkj6BHA6cF8Ht/GLdqbz2zqrvW209nhL7a3lzZ9va5mOKmT9QvO3NV/IdGeUM39X/ttpOu/8heco5PHu/n+3UVlG55Q0Fbie3OmcP4mIq1LaT120MDJdpXD+8qrk/JWcHZw/bWXp44+IB4AHSrCrBe0v0qU5f3lVcv5Kzg7On6qKGI/fzMyKx2P1mJlljAu/mVnGuPCbmWVMZgu/pB6SrpJ0k6SZ5c7TUZImSfq9pB9ImlTuPB0lqY+kOkknlDtLR0kanrzuSyVdUO48HSXpy5J+JOkOSV8sd56OknSwpIWSlpY7S6GSv/fFyet+VrnzVGThl/QTSa9KWtek/XhJf5X0jKQ57WzmRGAw8D65YSNKpkj5A3gLqKKE+YuUHeBS4M50UrauGPkj4smIOB84FZiQZt6mipT/nog4DzgfOC3NvE0VKf9zEXFuuknb18HncjKwNHndp5c8bFOd/XVcOW/AfwdqgHV5bT2BZ4GDgU8AT5Ab/XMk8Msmt/2BOcA/J+surcD8PZL1BgFLKiz7ceR+uDcLOKHSXvtknenAg8CZlZg/We97QE0F5y/p/9vdfC5zgTHJMj8tZ+6I6Lpj9bQlIn4nqbpJc+OonwCSbgdOjIj/CzTrTpC0GXgvmf0wvbTNFSN/nteAPdLI2ZIivfaTgD7k/kO8LemBiPgozdwNivXaR8R9wH2S7gd+ml7iZvstxusv4GrgwYj4c7qJd1Xkv/2y6shzIfepfDDwOF2gp6UiC38rWhr183NtLH83cJOkicDv0gxWoA7ll3Qy8D+AfYGb043Wrg5lj4h/A5A0C9hWqqLfho6+9pPIfXTfg9L8ELE9Hf3b/yZwLNBX0qER8YM0wxWgo69/f+Aq4EhJc5M3iK6itedyI3CzpGkUZ1iK3dKdCn+HRMQ/gLL3E3ZWRNxN7s2rYkXEonJn6IyIWAmsLHOMTouIG8kVoooUEdvJfT9RMSJiJ/C1cudoUPaPHEX0IvCZvPnBSVulqOT8lZwdnL/cKj1/vop4Lt2p8Bdj1M9yquT8lZwdnL/cKj1/vsp4LuX+drmT36b/DHiZj0/FPDdpnwo8Te5b9X8rd87umL+Sszt/+W+Vnr+7PBcP0mZmljHdqavHzMwK4MJvZpYxLvxmZhnjwm9mljEu/GZmGePCb2aWMS78ZmYZ48JvZZNcECQkDctrmyTpl0XY9iJJp7SzzCRJn+/gdidJekPS43m3Y3cvbZv7WyTpeUlFH5tG0kRJG5qOJ2/dnwu/ldMZwCPJfTlMAjpU+BO/j4gxebfl+Q8qp0eTtp6FbLiV5f41UhhBMyJ+T+5XppYxLvxWFpI+CXyB3Aippzd5eB9J9ydXMfqBcpfJ7Jkc/a6TtFbSxcl2xkj6o6Q1kpZJ2q+FfdVLGpBM10pamYyjfj5wcXLUPlHSQEl3SVqV3Aq+upak6iTvrcA64DOS3pL0PUlPAOMlTZH0lyT/TyTtkZfvGkl/Br7Szn4WSfp+8pyfSz6B/ETSk5IW5S33lqR/l7Re0nJJRyfP+zlJ5b8ClJWVC7+Vy4nAQxHxNLBd0ti8x44mN2b84cAh5Ma+HwN8OiKOiIiRwC3JsrcCl0bEKGAtcHkhO4+IeuAHwPzkqP33wA3J/FHADODHraw+sUlXzyFJ+1DgPyNiRERsInexmT9FxGigDlgEnJbk7wXkX693e0TURMTtBcTfDxgPXExuALD5wAhgpKQxyTJ9gN9ExAhgB3AluSufnQT8nwL2Yd2YC7+VyxlAQ5G7nV27ex6L3HVVPyQ3ENYXgOeAgyXdJOl44E1JfYF9I+K3yXqLyV0Or7OOJXexjMfJFdR9kk8mTTXt6nk2ad8UEX/MW+5D4K5k+jDg+eSNrqWsd3Qg5y8iN8jWWmBLRKyN3MVs1gPVyTLvAQ8l02uB30bE+8l0NZZpmb0Qi5WPpH7AMeSOUIPcdUpD0r8mizQdOTAi4jVJo8lddazhQucXF7jLD/j4IKeqjeV6AOMi4p0Ct9vUzibz7yRvXp1Zty3vJvcf5U03zDf8n34/Ph6BsXG5iPhIkv/fZ5yP+K0cTgFui4iDIqI6Ij4DPA9MTB4/OhnPvAdwGvBI0kffIyLuAr5D7iLhbwCvKXf5TICvAr+luXqgoStpRl77DmDvvPlfketiAnLfH+zOk2zir0C1pEOT+daymqXOhd/K4QxgWZO2u/i4u2cVuesIP0nuDWEZuWuZrky6Yf4LmJssOxP4d0lryH0P0FL/9RXADZLqyHW/NPgFcFLDl7vARUBt8kXxBlq/vF/TPv42TxsFSD5FfA34uaS15I7Cy32tW8soj8dv1oUlZ+r8MiKWprT96mT7R6SxfeuafMRv1rW9AXw3rR9wkfvUs63Y27auzUf8ZmYZ4yN+M7OMceE3M8sYF34zs4xx4TczyxgXfjOzjPn/PaG3RH0Pp9sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "plt.xscale(\"log\")\n",
    "plt.hist(nominal_errors, color=\"C0\", label=\"Nominal\")\n",
    "plt.hist(calibrated_errors, color=\"C1\", label=\"Calibrated\")\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel(\"Absolute Error [mm]\")\n",
    "plt.ylabel(\"Frequency\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}