streamingDataPipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "86c9df4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path=\"sample_data/sensor/aps_failure_training_set1.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4043e2c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6500a8c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.join(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b45f87c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7227bd90",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(file_path,na_values=\"na\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4e4e6b2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>aa_000</th>\n",
       "      <th>ab_000</th>\n",
       "      <th>ac_000</th>\n",
       "      <th>ad_000</th>\n",
       "      <th>ae_000</th>\n",
       "      <th>af_000</th>\n",
       "      <th>ag_000</th>\n",
       "      <th>ag_001</th>\n",
       "      <th>ag_002</th>\n",
       "      <th>...</th>\n",
       "      <th>ee_002</th>\n",
       "      <th>ee_003</th>\n",
       "      <th>ee_004</th>\n",
       "      <th>ee_005</th>\n",
       "      <th>ee_006</th>\n",
       "      <th>ee_007</th>\n",
       "      <th>ee_008</th>\n",
       "      <th>ee_009</th>\n",
       "      <th>ef_000</th>\n",
       "      <th>eg_000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pos</td>\n",
       "      <td>153204</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.820000e+02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>129862.0</td>\n",
       "      <td>26872.0</td>\n",
       "      <td>34044.0</td>\n",
       "      <td>22472.0</td>\n",
       "      <td>34362.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pos</td>\n",
       "      <td>453236</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.926000e+03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7908038.0</td>\n",
       "      <td>3026002.0</td>\n",
       "      <td>5025350.0</td>\n",
       "      <td>2025766.0</td>\n",
       "      <td>1160638.0</td>\n",
       "      <td>533834.0</td>\n",
       "      <td>493800.0</td>\n",
       "      <td>6914.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pos</td>\n",
       "      <td>72504</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.594000e+03</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>244.0</td>\n",
       "      <td>178226.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1432098.0</td>\n",
       "      <td>372252.0</td>\n",
       "      <td>527514.0</td>\n",
       "      <td>358274.0</td>\n",
       "      <td>332818.0</td>\n",
       "      <td>284178.0</td>\n",
       "      <td>3742.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pos</td>\n",
       "      <td>762958</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>776.0</td>\n",
       "      <td>281128.0</td>\n",
       "      <td>2186308.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pos</td>\n",
       "      <td>695994</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1397742.0</td>\n",
       "      <td>495544.0</td>\n",
       "      <td>361646.0</td>\n",
       "      <td>28610.0</td>\n",
       "      <td>5130.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36183</th>\n",
       "      <td>neg</td>\n",
       "      <td>153002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.640000e+02</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>998500.0</td>\n",
       "      <td>566884.0</td>\n",
       "      <td>1290398.0</td>\n",
       "      <td>1218244.0</td>\n",
       "      <td>1019768.0</td>\n",
       "      <td>717762.0</td>\n",
       "      <td>898642.0</td>\n",
       "      <td>28588.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36184</th>\n",
       "      <td>neg</td>\n",
       "      <td>2286</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.130707e+09</td>\n",
       "      <td>224.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10578.0</td>\n",
       "      <td>6760.0</td>\n",
       "      <td>21126.0</td>\n",
       "      <td>68424.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36185</th>\n",
       "      <td>neg</td>\n",
       "      <td>112</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>792.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>452.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>2622.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36186</th>\n",
       "      <td>neg</td>\n",
       "      <td>80292</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>494.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>699352.0</td>\n",
       "      <td>222654.0</td>\n",
       "      <td>347378.0</td>\n",
       "      <td>225724.0</td>\n",
       "      <td>194440.0</td>\n",
       "      <td>165070.0</td>\n",
       "      <td>802280.0</td>\n",
       "      <td>388422.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36187</th>\n",
       "      <td>neg</td>\n",
       "      <td>40222</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.980000e+02</td>\n",
       "      <td>628.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>440066.0</td>\n",
       "      <td>183200.0</td>\n",
       "      <td>344546.0</td>\n",
       "      <td>254068.0</td>\n",
       "      <td>225148.0</td>\n",
       "      <td>158304.0</td>\n",
       "      <td>170384.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36188 rows × 171 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      class  aa_000  ab_000        ac_000  ad_000  ae_000  af_000  ag_000  \\\n",
       "0       pos  153204     0.0  1.820000e+02     NaN     0.0     0.0     0.0   \n",
       "1       pos  453236     NaN  2.926000e+03     NaN     0.0     0.0     0.0   \n",
       "2       pos   72504     NaN  1.594000e+03  1052.0     0.0     0.0     0.0   \n",
       "3       pos  762958     NaN           NaN     NaN     NaN     NaN   776.0   \n",
       "4       pos  695994     NaN           NaN     NaN     NaN     NaN     0.0   \n",
       "...     ...     ...     ...           ...     ...     ...     ...     ...   \n",
       "36183   neg  153002     NaN  6.640000e+02   186.0     0.0     0.0     0.0   \n",
       "36184   neg    2286     NaN  2.130707e+09   224.0     0.0     0.0     0.0   \n",
       "36185   neg     112     0.0  2.130706e+09    18.0     0.0     0.0     0.0   \n",
       "36186   neg   80292     NaN  2.130706e+09   494.0     0.0     0.0     0.0   \n",
       "36187   neg   40222     NaN  6.980000e+02   628.0     0.0     0.0     0.0   \n",
       "\n",
       "         ag_001     ag_002  ...     ee_002     ee_003     ee_004     ee_005  \\\n",
       "0           0.0        0.0  ...   129862.0    26872.0    34044.0    22472.0   \n",
       "1           0.0      222.0  ...  7908038.0  3026002.0  5025350.0  2025766.0   \n",
       "2         244.0   178226.0  ...  1432098.0   372252.0   527514.0   358274.0   \n",
       "3      281128.0  2186308.0  ...        NaN        NaN        NaN        NaN   \n",
       "4           0.0        0.0  ...  1397742.0   495544.0   361646.0    28610.0   \n",
       "...         ...        ...  ...        ...        ...        ...        ...   \n",
       "36183       0.0        0.0  ...   998500.0   566884.0  1290398.0  1218244.0   \n",
       "36184       0.0        0.0  ...    10578.0     6760.0    21126.0    68424.0   \n",
       "36185       0.0        0.0  ...      792.0      386.0      452.0      144.0   \n",
       "36186       0.0        0.0  ...   699352.0   222654.0   347378.0   225724.0   \n",
       "36187       0.0        0.0  ...   440066.0   183200.0   344546.0   254068.0   \n",
       "\n",
       "          ee_006    ee_007    ee_008    ee_009  ef_000  eg_000  \n",
       "0        34362.0       0.0       0.0       0.0     0.0     0.0  \n",
       "1      1160638.0  533834.0  493800.0    6914.0     0.0     0.0  \n",
       "2       332818.0  284178.0    3742.0       0.0     0.0     0.0  \n",
       "3            NaN       NaN       NaN       NaN     NaN     NaN  \n",
       "4         5130.0     212.0       0.0       0.0     NaN     NaN  \n",
       "...          ...       ...       ...       ...     ...     ...  \n",
       "36183  1019768.0  717762.0  898642.0   28588.0     0.0     0.0  \n",
       "36184      136.0       0.0       0.0       0.0     0.0     0.0  \n",
       "36185      146.0    2622.0       0.0       0.0     0.0     0.0  \n",
       "36186   194440.0  165070.0  802280.0  388422.0     0.0     0.0  \n",
       "36187   225148.0  158304.0  170384.0     158.0     0.0     0.0  \n",
       "\n",
       "[36188 rows x 171 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e76e8789",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df[df.notna()].replace(\"na\",0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9d6ab2f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ff298ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "0b4f2d57",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "class Sensor:\n",
    "\n",
    "\n",
    "    def __init__(self, record: dict):\n",
    "        for k, v in record.items():\n",
    "            setattr(self, k, v)\n",
    "\n",
    "    @staticmethod\n",
    "    def dict_to_object(data: dict, ctx):\n",
    "        print(data, ctx)\n",
    "        return Sensor(record=data)\n",
    "\n",
    "    def to_dict(self):\n",
    "        return self.__dict__\n",
    "    @classmethod\n",
    "    def get_sensor_record(cls, file_path):\n",
    "        chunk_df = pd.read_csv(file_path,chunksize=10)\n",
    "        n_row = 0\n",
    "        for df in chunk_df:\n",
    "            for data in df.values:\n",
    "                sensor = Sensor(dict(zip(df.columns, data)))\n",
    "                # cars.append(car)\n",
    "                #print(n_row)\n",
    "                n_row += 1\n",
    "                yield sensor\n",
    "\n",
    "    @classmethod\n",
    "    def get_schema(cls,file_path):\n",
    "        columns = next(pd.read_csv(file_path,chunksize=10)).columns\n",
    "    \n",
    "        schema = dict()\n",
    "        schema.update({\n",
    "            \"$id\": \"http://example.com/myURI.schema.json\",\n",
    "            \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n",
    "            \"additionalProperties\": \"false\",\n",
    "            \"description\": \"Sample schema to help you get started.\",\n",
    "            'properties': dict(),\n",
    "            \"title\": \"SampleRecord\",\n",
    "            \"type\": \"object\"})\n",
    "        for column in columns:\n",
    "            schema['properties'].update(\n",
    "                {\n",
    "                    column: {\n",
    "                        \"description\": f\"sensor {column} \",\n",
    "                        \"type\": \"string\"\n",
    "                    }\n",
    "                }\n",
    "            )\n",
    "        return schema\n",
    "\n",
    "    def __str__(self):\n",
    "        return f\"{self.record}\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "cf181ab3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'$id': 'http://example.com/myURI.schema.json',\n",
       " '$schema': 'http://json-schema.org/draft-07/schema#',\n",
       " 'additionalProperties': 'false',\n",
       " 'description': 'Sample schema to help you get started.',\n",
       " 'properties': {'class': {'description': 'sensor class ', 'type': 'string'},\n",
       "  'aa_000': {'description': 'sensor aa_000 ', 'type': 'string'},\n",
       "  'ab_000': {'description': 'sensor ab_000 ', 'type': 'string'},\n",
       "  'ac_000': {'description': 'sensor ac_000 ', 'type': 'string'},\n",
       "  'ad_000': {'description': 'sensor ad_000 ', 'type': 'string'},\n",
       "  'ae_000': {'description': 'sensor ae_000 ', 'type': 'string'},\n",
       "  'af_000': {'description': 'sensor af_000 ', 'type': 'string'},\n",
       "  'ag_000': {'description': 'sensor ag_000 ', 'type': 'string'},\n",
       "  'ag_001': {'description': 'sensor ag_001 ', 'type': 'string'},\n",
       "  'ag_002': {'description': 'sensor ag_002 ', 'type': 'string'},\n",
       "  'ag_003': {'description': 'sensor ag_003 ', 'type': 'string'},\n",
       "  'ag_004': {'description': 'sensor ag_004 ', 'type': 'string'},\n",
       "  'ag_005': {'description': 'sensor ag_005 ', 'type': 'string'},\n",
       "  'ag_006': {'description': 'sensor ag_006 ', 'type': 'string'},\n",
       "  'ag_007': {'description': 'sensor ag_007 ', 'type': 'string'},\n",
       "  'ag_008': {'description': 'sensor ag_008 ', 'type': 'string'},\n",
       "  'ag_009': {'description': 'sensor ag_009 ', 'type': 'string'},\n",
       "  'ah_000': {'description': 'sensor ah_000 ', 'type': 'string'},\n",
       "  'ai_000': {'description': 'sensor ai_000 ', 'type': 'string'},\n",
       "  'aj_000': {'description': 'sensor aj_000 ', 'type': 'string'},\n",
       "  'ak_000': {'description': 'sensor ak_000 ', 'type': 'string'},\n",
       "  'al_000': {'description': 'sensor al_000 ', 'type': 'string'},\n",
       "  'am_0': {'description': 'sensor am_0 ', 'type': 'string'},\n",
       "  'an_000': {'description': 'sensor an_000 ', 'type': 'string'},\n",
       "  'ao_000': {'description': 'sensor ao_000 ', 'type': 'string'},\n",
       "  'ap_000': {'description': 'sensor ap_000 ', 'type': 'string'},\n",
       "  'aq_000': {'description': 'sensor aq_000 ', 'type': 'string'},\n",
       "  'ar_000': {'description': 'sensor ar_000 ', 'type': 'string'},\n",
       "  'as_000': {'description': 'sensor as_000 ', 'type': 'string'},\n",
       "  'at_000': {'description': 'sensor at_000 ', 'type': 'string'},\n",
       "  'au_000': {'description': 'sensor au_000 ', 'type': 'string'},\n",
       "  'av_000': {'description': 'sensor av_000 ', 'type': 'string'},\n",
       "  'ax_000': {'description': 'sensor ax_000 ', 'type': 'string'},\n",
       "  'ay_000': {'description': 'sensor ay_000 ', 'type': 'string'},\n",
       "  'ay_001': {'description': 'sensor ay_001 ', 'type': 'string'},\n",
       "  'ay_002': {'description': 'sensor ay_002 ', 'type': 'string'},\n",
       "  'ay_003': {'description': 'sensor ay_003 ', 'type': 'string'},\n",
       "  'ay_004': {'description': 'sensor ay_004 ', 'type': 'string'},\n",
       "  'ay_005': {'description': 'sensor ay_005 ', 'type': 'string'},\n",
       "  'ay_006': {'description': 'sensor ay_006 ', 'type': 'string'},\n",
       "  'ay_007': {'description': 'sensor ay_007 ', 'type': 'string'},\n",
       "  'ay_008': {'description': 'sensor ay_008 ', 'type': 'string'},\n",
       "  'ay_009': {'description': 'sensor ay_009 ', 'type': 'string'},\n",
       "  'az_000': {'description': 'sensor az_000 ', 'type': 'string'},\n",
       "  'az_001': {'description': 'sensor az_001 ', 'type': 'string'},\n",
       "  'az_002': {'description': 'sensor az_002 ', 'type': 'string'},\n",
       "  'az_003': {'description': 'sensor az_003 ', 'type': 'string'},\n",
       "  'az_004': {'description': 'sensor az_004 ', 'type': 'string'},\n",
       "  'az_005': {'description': 'sensor az_005 ', 'type': 'string'},\n",
       "  'az_006': {'description': 'sensor az_006 ', 'type': 'string'},\n",
       "  'az_007': {'description': 'sensor az_007 ', 'type': 'string'},\n",
       "  'az_008': {'description': 'sensor az_008 ', 'type': 'string'},\n",
       "  'az_009': {'description': 'sensor az_009 ', 'type': 'string'},\n",
       "  'ba_000': {'description': 'sensor ba_000 ', 'type': 'string'},\n",
       "  'ba_001': {'description': 'sensor ba_001 ', 'type': 'string'},\n",
       "  'ba_002': {'description': 'sensor ba_002 ', 'type': 'string'},\n",
       "  'ba_003': {'description': 'sensor ba_003 ', 'type': 'string'},\n",
       "  'ba_004': {'description': 'sensor ba_004 ', 'type': 'string'},\n",
       "  'ba_005': {'description': 'sensor ba_005 ', 'type': 'string'},\n",
       "  'ba_006': {'description': 'sensor ba_006 ', 'type': 'string'},\n",
       "  'ba_007': {'description': 'sensor ba_007 ', 'type': 'string'},\n",
       "  'ba_008': {'description': 'sensor ba_008 ', 'type': 'string'},\n",
       "  'ba_009': {'description': 'sensor ba_009 ', 'type': 'string'},\n",
       "  'bb_000': {'description': 'sensor bb_000 ', 'type': 'string'},\n",
       "  'bc_000': {'description': 'sensor bc_000 ', 'type': 'string'},\n",
       "  'bd_000': {'description': 'sensor bd_000 ', 'type': 'string'},\n",
       "  'be_000': {'description': 'sensor be_000 ', 'type': 'string'},\n",
       "  'bf_000': {'description': 'sensor bf_000 ', 'type': 'string'},\n",
       "  'bg_000': {'description': 'sensor bg_000 ', 'type': 'string'},\n",
       "  'bh_000': {'description': 'sensor bh_000 ', 'type': 'string'},\n",
       "  'bi_000': {'description': 'sensor bi_000 ', 'type': 'string'},\n",
       "  'bj_000': {'description': 'sensor bj_000 ', 'type': 'string'},\n",
       "  'bk_000': {'description': 'sensor bk_000 ', 'type': 'string'},\n",
       "  'bl_000': {'description': 'sensor bl_000 ', 'type': 'string'},\n",
       "  'bm_000': {'description': 'sensor bm_000 ', 'type': 'string'},\n",
       "  'bn_000': {'description': 'sensor bn_000 ', 'type': 'string'},\n",
       "  'bo_000': {'description': 'sensor bo_000 ', 'type': 'string'},\n",
       "  'bp_000': {'description': 'sensor bp_000 ', 'type': 'string'},\n",
       "  'bq_000': {'description': 'sensor bq_000 ', 'type': 'string'},\n",
       "  'br_000': {'description': 'sensor br_000 ', 'type': 'string'},\n",
       "  'bs_000': {'description': 'sensor bs_000 ', 'type': 'string'},\n",
       "  'bt_000': {'description': 'sensor bt_000 ', 'type': 'string'},\n",
       "  'bu_000': {'description': 'sensor bu_000 ', 'type': 'string'},\n",
       "  'bv_000': {'description': 'sensor bv_000 ', 'type': 'string'},\n",
       "  'bx_000': {'description': 'sensor bx_000 ', 'type': 'string'},\n",
       "  'by_000': {'description': 'sensor by_000 ', 'type': 'string'},\n",
       "  'bz_000': {'description': 'sensor bz_000 ', 'type': 'string'},\n",
       "  'ca_000': {'description': 'sensor ca_000 ', 'type': 'string'},\n",
       "  'cb_000': {'description': 'sensor cb_000 ', 'type': 'string'},\n",
       "  'cc_000': {'description': 'sensor cc_000 ', 'type': 'string'},\n",
       "  'cd_000': {'description': 'sensor cd_000 ', 'type': 'string'},\n",
       "  'ce_000': {'description': 'sensor ce_000 ', 'type': 'string'},\n",
       "  'cf_000': {'description': 'sensor cf_000 ', 'type': 'string'},\n",
       "  'cg_000': {'description': 'sensor cg_000 ', 'type': 'string'},\n",
       "  'ch_000': {'description': 'sensor ch_000 ', 'type': 'string'},\n",
       "  'ci_000': {'description': 'sensor ci_000 ', 'type': 'string'},\n",
       "  'cj_000': {'description': 'sensor cj_000 ', 'type': 'string'},\n",
       "  'ck_000': {'description': 'sensor ck_000 ', 'type': 'string'},\n",
       "  'cl_000': {'description': 'sensor cl_000 ', 'type': 'string'},\n",
       "  'cm_000': {'description': 'sensor cm_000 ', 'type': 'string'},\n",
       "  'cn_000': {'description': 'sensor cn_000 ', 'type': 'string'},\n",
       "  'cn_001': {'description': 'sensor cn_001 ', 'type': 'string'},\n",
       "  'cn_002': {'description': 'sensor cn_002 ', 'type': 'string'},\n",
       "  'cn_003': {'description': 'sensor cn_003 ', 'type': 'string'},\n",
       "  'cn_004': {'description': 'sensor cn_004 ', 'type': 'string'},\n",
       "  'cn_005': {'description': 'sensor cn_005 ', 'type': 'string'},\n",
       "  'cn_006': {'description': 'sensor cn_006 ', 'type': 'string'},\n",
       "  'cn_007': {'description': 'sensor cn_007 ', 'type': 'string'},\n",
       "  'cn_008': {'description': 'sensor cn_008 ', 'type': 'string'},\n",
       "  'cn_009': {'description': 'sensor cn_009 ', 'type': 'string'},\n",
       "  'co_000': {'description': 'sensor co_000 ', 'type': 'string'},\n",
       "  'cp_000': {'description': 'sensor cp_000 ', 'type': 'string'},\n",
       "  'cq_000': {'description': 'sensor cq_000 ', 'type': 'string'},\n",
       "  'cr_000': {'description': 'sensor cr_000 ', 'type': 'string'},\n",
       "  'cs_000': {'description': 'sensor cs_000 ', 'type': 'string'},\n",
       "  'cs_001': {'description': 'sensor cs_001 ', 'type': 'string'},\n",
       "  'cs_002': {'description': 'sensor cs_002 ', 'type': 'string'},\n",
       "  'cs_003': {'description': 'sensor cs_003 ', 'type': 'string'},\n",
       "  'cs_004': {'description': 'sensor cs_004 ', 'type': 'string'},\n",
       "  'cs_005': {'description': 'sensor cs_005 ', 'type': 'string'},\n",
       "  'cs_006': {'description': 'sensor cs_006 ', 'type': 'string'},\n",
       "  'cs_007': {'description': 'sensor cs_007 ', 'type': 'string'},\n",
       "  'cs_008': {'description': 'sensor cs_008 ', 'type': 'string'},\n",
       "  'cs_009': {'description': 'sensor cs_009 ', 'type': 'string'},\n",
       "  'ct_000': {'description': 'sensor ct_000 ', 'type': 'string'},\n",
       "  'cu_000': {'description': 'sensor cu_000 ', 'type': 'string'},\n",
       "  'cv_000': {'description': 'sensor cv_000 ', 'type': 'string'},\n",
       "  'cx_000': {'description': 'sensor cx_000 ', 'type': 'string'},\n",
       "  'cy_000': {'description': 'sensor cy_000 ', 'type': 'string'},\n",
       "  'cz_000': {'description': 'sensor cz_000 ', 'type': 'string'},\n",
       "  'da_000': {'description': 'sensor da_000 ', 'type': 'string'},\n",
       "  'db_000': {'description': 'sensor db_000 ', 'type': 'string'},\n",
       "  'dc_000': {'description': 'sensor dc_000 ', 'type': 'string'},\n",
       "  'dd_000': {'description': 'sensor dd_000 ', 'type': 'string'},\n",
       "  'de_000': {'description': 'sensor de_000 ', 'type': 'string'},\n",
       "  'df_000': {'description': 'sensor df_000 ', 'type': 'string'},\n",
       "  'dg_000': {'description': 'sensor dg_000 ', 'type': 'string'},\n",
       "  'dh_000': {'description': 'sensor dh_000 ', 'type': 'string'},\n",
       "  'di_000': {'description': 'sensor di_000 ', 'type': 'string'},\n",
       "  'dj_000': {'description': 'sensor dj_000 ', 'type': 'string'},\n",
       "  'dk_000': {'description': 'sensor dk_000 ', 'type': 'string'},\n",
       "  'dl_000': {'description': 'sensor dl_000 ', 'type': 'string'},\n",
       "  'dm_000': {'description': 'sensor dm_000 ', 'type': 'string'},\n",
       "  'dn_000': {'description': 'sensor dn_000 ', 'type': 'string'},\n",
       "  'do_000': {'description': 'sensor do_000 ', 'type': 'string'},\n",
       "  'dp_000': {'description': 'sensor dp_000 ', 'type': 'string'},\n",
       "  'dq_000': {'description': 'sensor dq_000 ', 'type': 'string'},\n",
       "  'dr_000': {'description': 'sensor dr_000 ', 'type': 'string'},\n",
       "  'ds_000': {'description': 'sensor ds_000 ', 'type': 'string'},\n",
       "  'dt_000': {'description': 'sensor dt_000 ', 'type': 'string'},\n",
       "  'du_000': {'description': 'sensor du_000 ', 'type': 'string'},\n",
       "  'dv_000': {'description': 'sensor dv_000 ', 'type': 'string'},\n",
       "  'dx_000': {'description': 'sensor dx_000 ', 'type': 'string'},\n",
       "  'dy_000': {'description': 'sensor dy_000 ', 'type': 'string'},\n",
       "  'dz_000': {'description': 'sensor dz_000 ', 'type': 'string'},\n",
       "  'ea_000': {'description': 'sensor ea_000 ', 'type': 'string'},\n",
       "  'eb_000': {'description': 'sensor eb_000 ', 'type': 'string'},\n",
       "  'ec_00': {'description': 'sensor ec_00 ', 'type': 'string'},\n",
       "  'ed_000': {'description': 'sensor ed_000 ', 'type': 'string'},\n",
       "  'ee_000': {'description': 'sensor ee_000 ', 'type': 'string'},\n",
       "  'ee_001': {'description': 'sensor ee_001 ', 'type': 'string'},\n",
       "  'ee_002': {'description': 'sensor ee_002 ', 'type': 'string'},\n",
       "  'ee_003': {'description': 'sensor ee_003 ', 'type': 'string'},\n",
       "  'ee_004': {'description': 'sensor ee_004 ', 'type': 'string'},\n",
       "  'ee_005': {'description': 'sensor ee_005 ', 'type': 'string'},\n",
       "  'ee_006': {'description': 'sensor ee_006 ', 'type': 'string'},\n",
       "  'ee_007': {'description': 'sensor ee_007 ', 'type': 'string'},\n",
       "  'ee_008': {'description': 'sensor ee_008 ', 'type': 'string'},\n",
       "  'ee_009': {'description': 'sensor ee_009 ', 'type': 'string'},\n",
       "  'ef_000': {'description': 'sensor ef_000 ', 'type': 'string'},\n",
       "  'eg_000': {'description': 'sensor eg_000 ', 'type': 'string'}},\n",
       " 'title': 'SampleRecord',\n",
       " 'type': 'object'}"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Sensor.get_schema(file_path=file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "6f624ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "sensor_record = Sensor.get_sensor_record(file_path=file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "fff352b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'class': 'neg', 'aa_000': 20, 'ab_000': '0', 'ac_000': 2130706436, 'ad_000': '20', 'ae_000': 10, 'af_000': 12, 'ag_000': 0, 'ag_001': 0, 'ag_002': 0, 'ag_003': 0, 'ag_004': 8, 'ag_005': 4484, 'ag_006': 3898, 'ag_007': 420, 'ag_008': 0, 'ag_009': 0, 'ah_000': 3282, 'ai_000': 0, 'aj_000': 0, 'ak_000': '0', 'al_000': 0, 'am_0': 0, 'an_000': 7348, 'ao_000': 4002, 'ap_000': 19076, 'aq_000': 1240, 'ar_000': 0, 'as_000': 0, 'at_000': 0, 'au_000': 0, 'av_000': 42, 'ax_000': 54, 'ay_000': 0, 'ay_001': 0, 'ay_002': 0, 'ay_003': 0, 'ay_004': 0, 'ay_005': 0, 'ay_006': 0, 'ay_007': 0, 'ay_008': 8810, 'ay_009': 0, 'az_000': 66, 'az_001': 14, 'az_002': 28, 'az_003': 584, 'az_004': 484, 'az_005': 6738, 'az_006': 896, 'az_007': 0, 'az_008': 0, 'az_009': 0, 'ba_000': 7182, 'ba_001': 580, 'ba_002': 304, 'ba_003': 126, 'ba_004': 88, 'ba_005': 386, 'ba_006': 140, 'ba_007': 2, 'ba_008': 2, 'ba_009': 0, 'bb_000': 26466, 'bc_000': 6, 'bd_000': 10, 'be_000': 0, 'bf_000': 0, 'bg_000': 3282, 'bh_000': 254, 'bi_000': 16032, 'bj_000': 2956, 'bk_000': 'na', 'bl_000': 'na', 'bm_000': 'na', 'bn_000': 'na', 'bo_000': 'na', 'bp_000': 'na', 'bq_000': 'na', 'br_000': 'na', 'bs_000': 18880, 'bt_000': 19.64, 'bu_000': 26466, 'bv_000': 26466, 'bx_000': '27886', 'by_000': 34, 'bz_000': 0, 'ca_000': '2488', 'cb_000': 26180, 'cc_000': '8870', 'cd_000': 1209600, 'ce_000': 534, 'cf_000': '2', 'cg_000': '8', 'ch_000': '0', 'ci_000': 4063.68, 'cj_000': 8.64, 'ck_000': 3772.8, 'cl_000': '0', 'cm_000': '14', 'cn_000': 0, 'cn_001': 0, 'cn_002': 0, 'cn_003': 2028, 'cn_004': 5322, 'cn_005': 1458, 'cn_006': 2, 'cn_007': 0, 'cn_008': 0, 'cn_009': 0, 'co_000': '12', 'cp_000': 12, 'cq_000': 26466, 'cr_000': '0', 'cs_000': 1060, 'cs_001': 8, 'cs_002': 10, 'cs_003': 96, 'cs_004': 1036, 'cs_005': 5298, 'cs_006': 1294, 'cs_007': 8, 'cs_008': 0, 'cs_009': 0, 'ct_000': '10', 'cu_000': '66', 'cv_000': '4282', 'cx_000': '94', 'cy_000': '0', 'cz_000': '0', 'da_000': '0', 'db_000': '0', 'dc_000': '4388', 'dd_000': 48, 'de_000': 48, 'df_000': '0', 'dg_000': '0', 'dh_000': '0', 'di_000': '12', 'dj_000': '0', 'dk_000': '0', 'dl_000': '0', 'dm_000': '0', 'dn_000': 228, 'do_000': 0, 'dp_000': 0, 'dq_000': 360, 'dr_000': 36, 'ds_000': 102, 'dt_000': 52, 'du_000': 6820, 'dv_000': 674, 'dx_000': 7362, 'dy_000': 46, 'dz_000': 0, 'ea_000': 0, 'eb_000': '0', 'ec_00': '1.28', 'ed_000': '50', 'ee_000': 5050, 'ee_001': 3392, 'ee_002': 162, 'ee_003': 28, 'ee_004': 72, 'ee_005': 28, 'ee_006': 56, 'ee_007': 22, 'ee_008': 0, 'ee_009': 0, 'ef_000': 0, 'eg_000': 0}\n",
      "{'class': 'neg', 'aa_000': 1068, 'ab_000': 'na', 'ac_000': '356', 'ad_000': '342', 'ae_000': 0, 'af_000': 0, 'ag_000': '0', 'ag_001': '0', 'ag_002': '0', 'ag_003': '0', 'ag_004': '118', 'ag_005': '23002', 'ag_006': '40522', 'ag_007': '1304', 'ag_008': '0', 'ag_009': '0', 'ah_000': 21518, 'ai_000': 0, 'aj_000': 0, 'ak_000': '0', 'al_000': 0, 'am_0': 0, 'an_000': 57456, 'ao_000': 48788, 'ap_000': 18600, 'aq_000': 5938, 'ar_000': '0', 'as_000': 0, 'at_000': 0, 'au_000': 0, 'av_000': 120, 'ax_000': 34, 'ay_000': '0', 'ay_001': '0', 'ay_002': '0', 'ay_003': '0', 'ay_004': '0', 'ay_005': '0', 'ay_006': '46444', 'ay_007': '18502', 'ay_008': '0', 'ay_009': '0', 'az_000': '144', 'az_001': '16', 'az_002': '46', 'az_003': '40', 'az_004': '5266', 'az_005': '59432', 'az_006': '2', 'az_007': '0', 'az_008': '0', 'az_009': '0', 'ba_000': '29588', 'ba_001': '23228', 'ba_002': '7916', 'ba_003': '2820', 'ba_004': '874', 'ba_005': '264', 'ba_006': '152', 'ba_007': '104', 'ba_008': '0', 'ba_009': '0', 'bb_000': 76092, 'bc_000': '14', 'bd_000': '14', 'be_000': 158, 'bf_000': 0, 'bg_000': 21518, 'bh_000': 990, 'bi_000': 7076, 'bj_000': 11496, 'bk_000': 'na', 'bl_000': 'na', 'bm_000': 'na', 'bn_000': 'na', 'bo_000': 'na', 'bp_000': 'na', 'bq_000': 'na', 'br_000': 'na', 'bs_000': 18320, 'bt_000': 1068.56, 'bu_000': 76092, 'bv_000': 76092, 'bx_000': '78050', 'by_000': '223', 'bz_000': '16', 'ca_000': '7418', 'cb_000': 75760, 'cc_000': '64946', 'cd_000': 1209600, 'ce_000': 2310, 'cf_000': '0', 'cg_000': '8', 'ch_000': '0', 'ci_000': 57167.04, 'cj_000': 0.0, 'ck_000': 7689.6, 'cl_000': '0', 'cm_000': '0', 'cn_000': '0', 'cn_001': '0', 'cn_002': '0', 'cn_003': '10258', 'cn_004': '45864', 'cn_005': '8518', 'cn_006': '246', 'cn_007': '48', 'cn_008': '12', 'cn_009': '0', 'co_000': '2', 'cp_000': '6', 'cq_000': 76092, 'cr_000': 'na', 'cs_000': '164', 'cs_001': '30', 'cs_002': '322', 'cs_003': '15156', 'cs_004': '1368', 'cs_005': '5356', 'cs_006': '39624', 'cs_007': '2858', 'cs_008': '68', 'cs_009': '0', 'ct_000': '548', 'cu_000': '152', 'cv_000': '56984', 'cx_000': '28726', 'cy_000': '0', 'cz_000': '12', 'da_000': '0', 'db_000': '52', 'dc_000': '58346', 'dd_000': 104, 'de_000': '50', 'df_000': '0', 'dg_000': '0', 'dh_000': '0', 'di_000': '0', 'dj_000': '0', 'dk_000': '0', 'dl_000': '0', 'dm_000': '0', 'dn_000': 486, 'do_000': '510', 'dp_000': '144', 'dq_000': '0', 'dr_000': '0', 'ds_000': '2380', 'dt_000': '372', 'du_000': '6380', 'dv_000': '752', 'dx_000': '0', 'dy_000': '0', 'dz_000': '0', 'ea_000': '0', 'eb_000': '68580', 'ec_00': '116.92', 'ed_000': '68', 'ee_000': '8616', 'ee_001': '7340', 'ee_002': '2776', 'ee_003': '1950', 'ee_004': '10574', 'ee_005': '33340', 'ee_006': '338', 'ee_007': '12', 'ee_008': '0', 'ee_009': '0', 'ef_000': '0', 'eg_000': '0'}\n",
      "{'class': 'neg', 'aa_000': 31570, 'ab_000': 'na', 'ac_000': '0', 'ad_000': 'na', 'ae_000': '0', 'af_000': '0', 'ag_000': 0, 'ag_001': 0, 'ag_002': 0, 'ag_003': 0, 'ag_004': 1396, 'ag_005': 106278, 'ag_006': 1388552, 'ag_007': 857536, 'ag_008': 80002, 'ag_009': 2456, 'ah_000': 1219492, 'ai_000': 0, 'aj_000': 0, 'ak_000': '0', 'al_000': 0, 'am_0': 0, 'an_000': 2179528, 'ao_000': 1946096, 'ap_000': 349314, 'aq_000': 195836, 'ar_000': '0', 'as_000': 0, 'at_000': 9188, 'au_000': 0, 'av_000': '0', 'ax_000': '0', 'ay_000': 0, 'ay_001': 0, 'ay_002': 0, 'ay_003': 0, 'ay_004': 0, 'ay_005': 6470, 'ay_006': 850552, 'ay_007': 354044, 'ay_008': 1225154, 'ay_009': 0, 'az_000': 10968, 'az_001': 24876, 'az_002': 19038, 'az_003': 219486, 'az_004': 1855262, 'az_005': 305392, 'az_006': 1198, 'az_007': 0, 'az_008': 0, 'az_009': 0, 'ba_000': 982100, 'ba_001': 306914, 'ba_002': 147058, 'ba_003': 110952, 'ba_004': 94762, 'ba_005': 99536, 'ba_006': 157304, 'ba_007': 371542, 'ba_008': 161110, 'ba_009': 4942, 'bb_000': 2529312, 'bc_000': '30', 'bd_000': '36', 'be_000': '254', 'bf_000': '46', 'bg_000': 1219492, 'bh_000': 30852, 'bi_000': 161846, 'bj_000': 186372, 'bk_000': '168760', 'bl_000': '162540', 'bm_000': 'na', 'bn_000': 'na', 'bo_000': 'na', 'bp_000': 'na', 'bq_000': 'na', 'br_000': 'na', 'bs_000': 13420, 'bt_000': 31569.38, 'bu_000': 2529312, 'bv_000': 2529312, 'bx_000': '2510248', 'by_000': 20216, 'bz_000': '991084', 'ca_000': '8152', 'cb_000': 104680, 'cc_000': '2436220', 'cd_000': 1209600, 'ce_000': '0', 'cf_000': 'na', 'cg_000': 'na', 'ch_000': 'na', 'ci_000': 1872828.48, 'cj_000': 0.0, 'ck_000': 558646.08, 'cl_000': 'na', 'cm_000': 'na', 'cn_000': 0, 'cn_001': 0, 'cn_002': 0, 'cn_003': 2362, 'cn_004': 316344, 'cn_005': 1100614, 'cn_006': 910592, 'cn_007': 92108, 'cn_008': 12570, 'cn_009': 1630, 'co_000': 'na', 'cp_000': '434', 'cq_000': 2529312, 'cr_000': 'na', 'cs_000': 4276, 'cs_001': 620, 'cs_002': 259544, 'cs_003': 261782, 'cs_004': 357358, 'cs_005': 1483212, 'cs_006': 67464, 'cs_007': 1964, 'cs_008': 0, 'cs_009': 0, 'ct_000': 'na', 'cu_000': 'na', 'cv_000': 'na', 'cx_000': 'na', 'cy_000': 'na', 'cz_000': 'na', 'da_000': 'na', 'db_000': 'na', 'dc_000': 'na', 'dd_000': '1740', 'de_000': '98', 'df_000': '0', 'dg_000': '0', 'dh_000': '0', 'di_000': '0', 'dj_000': '0', 'dk_000': '0', 'dl_000': '0', 'dm_000': '0', 'dn_000': 19640, 'do_000': '28474', 'dp_000': '5722', 'dq_000': '0', 'dr_000': '0', 'ds_000': '52560', 'dt_000': '9786', 'du_000': '1496180', 'dv_000': '918484', 'dx_000': '0', 'dy_000': '8070', 'dz_000': '0', 'ea_000': '0', 'eb_000': '30569400', 'ec_00': 'na', 'ed_000': 'na', 'ee_000': 576516, 'ee_001': 490770, 'ee_002': 241462, 'ee_003': 123836, 'ee_004': 251764, 'ee_005': 228118, 'ee_006': 241510, 'ee_007': 144050, 'ee_008': 134192, 'ee_009': 4002, 'ef_000': '0', 'eg_000': '0'}\n",
      "36188\n"
     ]
    }
   ],
   "source": [
    "n_r=0\n",
    "for record in sensor_record:\n",
    "    n_r+=1\n",
    "    if n_r%10000==0:\n",
    "        print(record.to_dict())\n",
    "print(n_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5d5099b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>aa_000</th>\n",
       "      <th>ab_000</th>\n",
       "      <th>ac_000</th>\n",
       "      <th>ad_000</th>\n",
       "      <th>ae_000</th>\n",
       "      <th>af_000</th>\n",
       "      <th>ag_000</th>\n",
       "      <th>ag_001</th>\n",
       "      <th>ag_002</th>\n",
       "      <th>...</th>\n",
       "      <th>ee_002</th>\n",
       "      <th>ee_003</th>\n",
       "      <th>ee_004</th>\n",
       "      <th>ee_005</th>\n",
       "      <th>ee_006</th>\n",
       "      <th>ee_007</th>\n",
       "      <th>ee_008</th>\n",
       "      <th>ee_009</th>\n",
       "      <th>ef_000</th>\n",
       "      <th>eg_000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pos</td>\n",
       "      <td>153204</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>129862</td>\n",
       "      <td>26872</td>\n",
       "      <td>34044</td>\n",
       "      <td>22472</td>\n",
       "      <td>34362</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pos</td>\n",
       "      <td>453236</td>\n",
       "      <td>0</td>\n",
       "      <td>2926</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>222</td>\n",
       "      <td>...</td>\n",
       "      <td>7908038</td>\n",
       "      <td>3026002</td>\n",
       "      <td>5025350</td>\n",
       "      <td>2025766</td>\n",
       "      <td>1160638</td>\n",
       "      <td>533834</td>\n",
       "      <td>493800</td>\n",
       "      <td>6914</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pos</td>\n",
       "      <td>72504</td>\n",
       "      <td>0</td>\n",
       "      <td>1594</td>\n",
       "      <td>1052</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>244</td>\n",
       "      <td>178226</td>\n",
       "      <td>...</td>\n",
       "      <td>1432098</td>\n",
       "      <td>372252</td>\n",
       "      <td>527514</td>\n",
       "      <td>358274</td>\n",
       "      <td>332818</td>\n",
       "      <td>284178</td>\n",
       "      <td>3742</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pos</td>\n",
       "      <td>762958</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>776</td>\n",
       "      <td>281128</td>\n",
       "      <td>2186308</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pos</td>\n",
       "      <td>695994</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1397742</td>\n",
       "      <td>495544</td>\n",
       "      <td>361646</td>\n",
       "      <td>28610</td>\n",
       "      <td>5130</td>\n",
       "      <td>212</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36183</th>\n",
       "      <td>neg</td>\n",
       "      <td>153002</td>\n",
       "      <td>0</td>\n",
       "      <td>664</td>\n",
       "      <td>186</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>998500</td>\n",
       "      <td>566884</td>\n",
       "      <td>1290398</td>\n",
       "      <td>1218244</td>\n",
       "      <td>1019768</td>\n",
       "      <td>717762</td>\n",
       "      <td>898642</td>\n",
       "      <td>28588</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36184</th>\n",
       "      <td>neg</td>\n",
       "      <td>2286</td>\n",
       "      <td>0</td>\n",
       "      <td>2130706538</td>\n",
       "      <td>224</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>10578</td>\n",
       "      <td>6760</td>\n",
       "      <td>21126</td>\n",
       "      <td>68424</td>\n",
       "      <td>136</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36185</th>\n",
       "      <td>neg</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "      <td>2130706432</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>792</td>\n",
       "      <td>386</td>\n",
       "      <td>452</td>\n",
       "      <td>144</td>\n",
       "      <td>146</td>\n",
       "      <td>2622</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36186</th>\n",
       "      <td>neg</td>\n",
       "      <td>80292</td>\n",
       "      <td>0</td>\n",
       "      <td>2130706432</td>\n",
       "      <td>494</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>699352</td>\n",
       "      <td>222654</td>\n",
       "      <td>347378</td>\n",
       "      <td>225724</td>\n",
       "      <td>194440</td>\n",
       "      <td>165070</td>\n",
       "      <td>802280</td>\n",
       "      <td>388422</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36187</th>\n",
       "      <td>neg</td>\n",
       "      <td>40222</td>\n",
       "      <td>0</td>\n",
       "      <td>698</td>\n",
       "      <td>628</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>440066</td>\n",
       "      <td>183200</td>\n",
       "      <td>344546</td>\n",
       "      <td>254068</td>\n",
       "      <td>225148</td>\n",
       "      <td>158304</td>\n",
       "      <td>170384</td>\n",
       "      <td>158</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36188 rows × 171 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      class  aa_000 ab_000      ac_000 ad_000 ae_000 af_000 ag_000  ag_001  \\\n",
       "0       pos  153204      0         182      0      0      0      0       0   \n",
       "1       pos  453236      0        2926      0      0      0      0       0   \n",
       "2       pos   72504      0        1594   1052      0      0      0     244   \n",
       "3       pos  762958      0           0      0      0      0    776  281128   \n",
       "4       pos  695994      0           0      0      0      0      0       0   \n",
       "...     ...     ...    ...         ...    ...    ...    ...    ...     ...   \n",
       "36183   neg  153002      0         664    186      0      0      0       0   \n",
       "36184   neg    2286      0  2130706538    224      0      0      0       0   \n",
       "36185   neg     112      0  2130706432     18      0      0      0       0   \n",
       "36186   neg   80292      0  2130706432    494      0      0      0       0   \n",
       "36187   neg   40222      0         698    628      0      0      0       0   \n",
       "\n",
       "        ag_002  ...   ee_002   ee_003   ee_004   ee_005   ee_006  ee_007  \\\n",
       "0            0  ...   129862    26872    34044    22472    34362       0   \n",
       "1          222  ...  7908038  3026002  5025350  2025766  1160638  533834   \n",
       "2       178226  ...  1432098   372252   527514   358274   332818  284178   \n",
       "3      2186308  ...        0        0        0        0        0       0   \n",
       "4            0  ...  1397742   495544   361646    28610     5130     212   \n",
       "...        ...  ...      ...      ...      ...      ...      ...     ...   \n",
       "36183        0  ...   998500   566884  1290398  1218244  1019768  717762   \n",
       "36184        0  ...    10578     6760    21126    68424      136       0   \n",
       "36185        0  ...      792      386      452      144      146    2622   \n",
       "36186        0  ...   699352   222654   347378   225724   194440  165070   \n",
       "36187        0  ...   440066   183200   344546   254068   225148  158304   \n",
       "\n",
       "       ee_008  ee_009 ef_000 eg_000  \n",
       "0           0       0      0      0  \n",
       "1      493800    6914      0      0  \n",
       "2        3742       0      0      0  \n",
       "3           0       0      0      0  \n",
       "4           0       0      0      0  \n",
       "...       ...     ...    ...    ...  \n",
       "36183  898642   28588      0      0  \n",
       "36184       0       0      0      0  \n",
       "36185       0       0      0      0  \n",
       "36186  802280  388422      0      0  \n",
       "36187  170384     158      0      0  \n",
       "\n",
       "[36188 rows x 171 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "2f8b052b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "d5080a27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'$id': 'http://example.com/myURI.schema.json',\n",
       " '$schema': 'http://json-schema.org/draft-07/schema#',\n",
       " 'additionalProperties': 'false',\n",
       " 'description': 'Sample schema to help you get started.',\n",
       " 'properties': {'class': {'description': 'sensor class ', 'type': 'string'},\n",
       "  'aa_000': {'description': 'sensor aa_000 ', 'type': 'string'},\n",
       "  'ab_000': {'description': 'sensor ab_000 ', 'type': 'string'},\n",
       "  'ac_000': {'description': 'sensor ac_000 ', 'type': 'string'},\n",
       "  'ad_000': {'description': 'sensor ad_000 ', 'type': 'string'},\n",
       "  'ae_000': {'description': 'sensor ae_000 ', 'type': 'string'},\n",
       "  'af_000': {'description': 'sensor af_000 ', 'type': 'string'},\n",
       "  'ag_000': {'description': 'sensor ag_000 ', 'type': 'string'},\n",
       "  'ag_001': {'description': 'sensor ag_001 ', 'type': 'string'},\n",
       "  'ag_002': {'description': 'sensor ag_002 ', 'type': 'string'},\n",
       "  'ag_003': {'description': 'sensor ag_003 ', 'type': 'string'},\n",
       "  'ag_004': {'description': 'sensor ag_004 ', 'type': 'string'},\n",
       "  'ag_005': {'description': 'sensor ag_005 ', 'type': 'string'},\n",
       "  'ag_006': {'description': 'sensor ag_006 ', 'type': 'string'},\n",
       "  'ag_007': {'description': 'sensor ag_007 ', 'type': 'string'},\n",
       "  'ag_008': {'description': 'sensor ag_008 ', 'type': 'string'},\n",
       "  'ag_009': {'description': 'sensor ag_009 ', 'type': 'string'},\n",
       "  'ah_000': {'description': 'sensor ah_000 ', 'type': 'string'},\n",
       "  'ai_000': {'description': 'sensor ai_000 ', 'type': 'string'},\n",
       "  'aj_000': {'description': 'sensor aj_000 ', 'type': 'string'},\n",
       "  'ak_000': {'description': 'sensor ak_000 ', 'type': 'string'},\n",
       "  'al_000': {'description': 'sensor al_000 ', 'type': 'string'},\n",
       "  'am_0': {'description': 'sensor am_0 ', 'type': 'string'},\n",
       "  'an_000': {'description': 'sensor an_000 ', 'type': 'string'},\n",
       "  'ao_000': {'description': 'sensor ao_000 ', 'type': 'string'},\n",
       "  'ap_000': {'description': 'sensor ap_000 ', 'type': 'string'},\n",
       "  'aq_000': {'description': 'sensor aq_000 ', 'type': 'string'},\n",
       "  'ar_000': {'description': 'sensor ar_000 ', 'type': 'string'},\n",
       "  'as_000': {'description': 'sensor as_000 ', 'type': 'string'},\n",
       "  'at_000': {'description': 'sensor at_000 ', 'type': 'string'},\n",
       "  'au_000': {'description': 'sensor au_000 ', 'type': 'string'},\n",
       "  'av_000': {'description': 'sensor av_000 ', 'type': 'string'},\n",
       "  'ax_000': {'description': 'sensor ax_000 ', 'type': 'string'},\n",
       "  'ay_000': {'description': 'sensor ay_000 ', 'type': 'string'},\n",
       "  'ay_001': {'description': 'sensor ay_001 ', 'type': 'string'},\n",
       "  'ay_002': {'description': 'sensor ay_002 ', 'type': 'string'},\n",
       "  'ay_003': {'description': 'sensor ay_003 ', 'type': 'string'},\n",
       "  'ay_004': {'description': 'sensor ay_004 ', 'type': 'string'},\n",
       "  'ay_005': {'description': 'sensor ay_005 ', 'type': 'string'},\n",
       "  'ay_006': {'description': 'sensor ay_006 ', 'type': 'string'},\n",
       "  'ay_007': {'description': 'sensor ay_007 ', 'type': 'string'},\n",
       "  'ay_008': {'description': 'sensor ay_008 ', 'type': 'string'},\n",
       "  'ay_009': {'description': 'sensor ay_009 ', 'type': 'string'},\n",
       "  'az_000': {'description': 'sensor az_000 ', 'type': 'string'},\n",
       "  'az_001': {'description': 'sensor az_001 ', 'type': 'string'},\n",
       "  'az_002': {'description': 'sensor az_002 ', 'type': 'string'},\n",
       "  'az_003': {'description': 'sensor az_003 ', 'type': 'string'},\n",
       "  'az_004': {'description': 'sensor az_004 ', 'type': 'string'},\n",
       "  'az_005': {'description': 'sensor az_005 ', 'type': 'string'},\n",
       "  'az_006': {'description': 'sensor az_006 ', 'type': 'string'},\n",
       "  'az_007': {'description': 'sensor az_007 ', 'type': 'string'},\n",
       "  'az_008': {'description': 'sensor az_008 ', 'type': 'string'},\n",
       "  'az_009': {'description': 'sensor az_009 ', 'type': 'string'},\n",
       "  'ba_000': {'description': 'sensor ba_000 ', 'type': 'string'},\n",
       "  'ba_001': {'description': 'sensor ba_001 ', 'type': 'string'},\n",
       "  'ba_002': {'description': 'sensor ba_002 ', 'type': 'string'},\n",
       "  'ba_003': {'description': 'sensor ba_003 ', 'type': 'string'},\n",
       "  'ba_004': {'description': 'sensor ba_004 ', 'type': 'string'},\n",
       "  'ba_005': {'description': 'sensor ba_005 ', 'type': 'string'},\n",
       "  'ba_006': {'description': 'sensor ba_006 ', 'type': 'string'},\n",
       "  'ba_007': {'description': 'sensor ba_007 ', 'type': 'string'},\n",
       "  'ba_008': {'description': 'sensor ba_008 ', 'type': 'string'},\n",
       "  'ba_009': {'description': 'sensor ba_009 ', 'type': 'string'},\n",
       "  'bb_000': {'description': 'sensor bb_000 ', 'type': 'string'},\n",
       "  'bc_000': {'description': 'sensor bc_000 ', 'type': 'string'},\n",
       "  'bd_000': {'description': 'sensor bd_000 ', 'type': 'string'},\n",
       "  'be_000': {'description': 'sensor be_000 ', 'type': 'string'},\n",
       "  'bf_000': {'description': 'sensor bf_000 ', 'type': 'string'},\n",
       "  'bg_000': {'description': 'sensor bg_000 ', 'type': 'string'},\n",
       "  'bh_000': {'description': 'sensor bh_000 ', 'type': 'string'},\n",
       "  'bi_000': {'description': 'sensor bi_000 ', 'type': 'string'},\n",
       "  'bj_000': {'description': 'sensor bj_000 ', 'type': 'string'},\n",
       "  'bk_000': {'description': 'sensor bk_000 ', 'type': 'string'},\n",
       "  'bl_000': {'description': 'sensor bl_000 ', 'type': 'string'},\n",
       "  'bm_000': {'description': 'sensor bm_000 ', 'type': 'string'},\n",
       "  'bn_000': {'description': 'sensor bn_000 ', 'type': 'string'},\n",
       "  'bo_000': {'description': 'sensor bo_000 ', 'type': 'string'},\n",
       "  'bp_000': {'description': 'sensor bp_000 ', 'type': 'string'},\n",
       "  'bq_000': {'description': 'sensor bq_000 ', 'type': 'string'},\n",
       "  'br_000': {'description': 'sensor br_000 ', 'type': 'string'},\n",
       "  'bs_000': {'description': 'sensor bs_000 ', 'type': 'string'},\n",
       "  'bt_000': {'description': 'sensor bt_000 ', 'type': 'string'},\n",
       "  'bu_000': {'description': 'sensor bu_000 ', 'type': 'string'},\n",
       "  'bv_000': {'description': 'sensor bv_000 ', 'type': 'string'},\n",
       "  'bx_000': {'description': 'sensor bx_000 ', 'type': 'string'},\n",
       "  'by_000': {'description': 'sensor by_000 ', 'type': 'string'},\n",
       "  'bz_000': {'description': 'sensor bz_000 ', 'type': 'string'},\n",
       "  'ca_000': {'description': 'sensor ca_000 ', 'type': 'string'},\n",
       "  'cb_000': {'description': 'sensor cb_000 ', 'type': 'string'},\n",
       "  'cc_000': {'description': 'sensor cc_000 ', 'type': 'string'},\n",
       "  'cd_000': {'description': 'sensor cd_000 ', 'type': 'string'},\n",
       "  'ce_000': {'description': 'sensor ce_000 ', 'type': 'string'},\n",
       "  'cf_000': {'description': 'sensor cf_000 ', 'type': 'string'},\n",
       "  'cg_000': {'description': 'sensor cg_000 ', 'type': 'string'},\n",
       "  'ch_000': {'description': 'sensor ch_000 ', 'type': 'string'},\n",
       "  'ci_000': {'description': 'sensor ci_000 ', 'type': 'string'},\n",
       "  'cj_000': {'description': 'sensor cj_000 ', 'type': 'string'},\n",
       "  'ck_000': {'description': 'sensor ck_000 ', 'type': 'string'},\n",
       "  'cl_000': {'description': 'sensor cl_000 ', 'type': 'string'},\n",
       "  'cm_000': {'description': 'sensor cm_000 ', 'type': 'string'},\n",
       "  'cn_000': {'description': 'sensor cn_000 ', 'type': 'string'},\n",
       "  'cn_001': {'description': 'sensor cn_001 ', 'type': 'string'},\n",
       "  'cn_002': {'description': 'sensor cn_002 ', 'type': 'string'},\n",
       "  'cn_003': {'description': 'sensor cn_003 ', 'type': 'string'},\n",
       "  'cn_004': {'description': 'sensor cn_004 ', 'type': 'string'},\n",
       "  'cn_005': {'description': 'sensor cn_005 ', 'type': 'string'},\n",
       "  'cn_006': {'description': 'sensor cn_006 ', 'type': 'string'},\n",
       "  'cn_007': {'description': 'sensor cn_007 ', 'type': 'string'},\n",
       "  'cn_008': {'description': 'sensor cn_008 ', 'type': 'string'},\n",
       "  'cn_009': {'description': 'sensor cn_009 ', 'type': 'string'},\n",
       "  'co_000': {'description': 'sensor co_000 ', 'type': 'string'},\n",
       "  'cp_000': {'description': 'sensor cp_000 ', 'type': 'string'},\n",
       "  'cq_000': {'description': 'sensor cq_000 ', 'type': 'string'},\n",
       "  'cr_000': {'description': 'sensor cr_000 ', 'type': 'string'},\n",
       "  'cs_000': {'description': 'sensor cs_000 ', 'type': 'string'},\n",
       "  'cs_001': {'description': 'sensor cs_001 ', 'type': 'string'},\n",
       "  'cs_002': {'description': 'sensor cs_002 ', 'type': 'string'},\n",
       "  'cs_003': {'description': 'sensor cs_003 ', 'type': 'string'},\n",
       "  'cs_004': {'description': 'sensor cs_004 ', 'type': 'string'},\n",
       "  'cs_005': {'description': 'sensor cs_005 ', 'type': 'string'},\n",
       "  'cs_006': {'description': 'sensor cs_006 ', 'type': 'string'},\n",
       "  'cs_007': {'description': 'sensor cs_007 ', 'type': 'string'},\n",
       "  'cs_008': {'description': 'sensor cs_008 ', 'type': 'string'},\n",
       "  'cs_009': {'description': 'sensor cs_009 ', 'type': 'string'},\n",
       "  'ct_000': {'description': 'sensor ct_000 ', 'type': 'string'},\n",
       "  'cu_000': {'description': 'sensor cu_000 ', 'type': 'string'},\n",
       "  'cv_000': {'description': 'sensor cv_000 ', 'type': 'string'},\n",
       "  'cx_000': {'description': 'sensor cx_000 ', 'type': 'string'},\n",
       "  'cy_000': {'description': 'sensor cy_000 ', 'type': 'string'},\n",
       "  'cz_000': {'description': 'sensor cz_000 ', 'type': 'string'},\n",
       "  'da_000': {'description': 'sensor da_000 ', 'type': 'string'},\n",
       "  'db_000': {'description': 'sensor db_000 ', 'type': 'string'},\n",
       "  'dc_000': {'description': 'sensor dc_000 ', 'type': 'string'},\n",
       "  'dd_000': {'description': 'sensor dd_000 ', 'type': 'string'},\n",
       "  'de_000': {'description': 'sensor de_000 ', 'type': 'string'},\n",
       "  'df_000': {'description': 'sensor df_000 ', 'type': 'string'},\n",
       "  'dg_000': {'description': 'sensor dg_000 ', 'type': 'string'},\n",
       "  'dh_000': {'description': 'sensor dh_000 ', 'type': 'string'},\n",
       "  'di_000': {'description': 'sensor di_000 ', 'type': 'string'},\n",
       "  'dj_000': {'description': 'sensor dj_000 ', 'type': 'string'},\n",
       "  'dk_000': {'description': 'sensor dk_000 ', 'type': 'string'},\n",
       "  'dl_000': {'description': 'sensor dl_000 ', 'type': 'string'},\n",
       "  'dm_000': {'description': 'sensor dm_000 ', 'type': 'string'},\n",
       "  'dn_000': {'description': 'sensor dn_000 ', 'type': 'string'},\n",
       "  'do_000': {'description': 'sensor do_000 ', 'type': 'string'},\n",
       "  'dp_000': {'description': 'sensor dp_000 ', 'type': 'string'},\n",
       "  'dq_000': {'description': 'sensor dq_000 ', 'type': 'string'},\n",
       "  'dr_000': {'description': 'sensor dr_000 ', 'type': 'string'},\n",
       "  'ds_000': {'description': 'sensor ds_000 ', 'type': 'string'},\n",
       "  'dt_000': {'description': 'sensor dt_000 ', 'type': 'string'},\n",
       "  'du_000': {'description': 'sensor du_000 ', 'type': 'string'},\n",
       "  'dv_000': {'description': 'sensor dv_000 ', 'type': 'string'},\n",
       "  'dx_000': {'description': 'sensor dx_000 ', 'type': 'string'},\n",
       "  'dy_000': {'description': 'sensor dy_000 ', 'type': 'string'},\n",
       "  'dz_000': {'description': 'sensor dz_000 ', 'type': 'string'},\n",
       "  'ea_000': {'description': 'sensor ea_000 ', 'type': 'string'},\n",
       "  'eb_000': {'description': 'sensor eb_000 ', 'type': 'string'},\n",
       "  'ec_00': {'description': 'sensor ec_00 ', 'type': 'string'},\n",
       "  'ed_000': {'description': 'sensor ed_000 ', 'type': 'string'},\n",
       "  'ee_000': {'description': 'sensor ee_000 ', 'type': 'string'},\n",
       "  'ee_001': {'description': 'sensor ee_001 ', 'type': 'string'},\n",
       "  'ee_002': {'description': 'sensor ee_002 ', 'type': 'string'},\n",
       "  'ee_003': {'description': 'sensor ee_003 ', 'type': 'string'},\n",
       "  'ee_004': {'description': 'sensor ee_004 ', 'type': 'string'},\n",
       "  'ee_005': {'description': 'sensor ee_005 ', 'type': 'string'},\n",
       "  'ee_006': {'description': 'sensor ee_006 ', 'type': 'string'},\n",
       "  'ee_007': {'description': 'sensor ee_007 ', 'type': 'string'},\n",
       "  'ee_008': {'description': 'sensor ee_008 ', 'type': 'string'},\n",
       "  'ee_009': {'description': 'sensor ee_009 ', 'type': 'string'},\n",
       "  'ef_000': {'description': 'sensor ef_000 ', 'type': 'string'},\n",
       "  'eg_000': {'description': 'sensor eg_000 ', 'type': 'string'}},\n",
       " 'title': 'SampleRecord',\n",
       " 'type': 'object'}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_schema(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49b55475",
   "metadata": {},
   "outputs": [],
   "source": [
    "column_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "6277efbe",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "illegal target for annotation (136549307.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Input \u001b[0;32mIn [59]\u001b[0;36m\u001b[0m\n\u001b[0;31m    \"brand\": {\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m illegal target for annotation\n"
     ]
    }
   ],
   "source": [
    "{\"brand\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    }}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "fb919469",
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_str = \"\"\"\n",
    "    {\n",
    "  \"$id\": \"http://example.com/myURI.schema.json\",\n",
    "  \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n",
    "  \"additionalProperties\": false,\n",
    "  \"description\": \"Sample schema to help you get started.\",\n",
    "  \"properties\": {\n",
    "    \"brand\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    },\n",
    "    \"car_name\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    },\n",
    "    \"engine\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    },\n",
    "    \"fuel_type\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    },\n",
    "    \"km_driven\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    },\n",
    "    \"max_power\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    },\n",
    "    \"mileage\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    },\n",
    "    \"model\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    },\n",
    "    \"seats\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    },\n",
    "    \"seller_type\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    },\n",
    "    \"selling_price\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    },\n",
    "    \"transmission_type\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"string\"\n",
    "    },\n",
    "    \"vehicle_age\": {\n",
    "      \"description\": \"The type(v) type is used.\",\n",
    "      \"type\": \"number\"\n",
    "    }\n",
    "  },\n",
    "  \"title\": \"SampleRecord\",\n",
    "  \"type\": \"object\"\n",
    "}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "4371cbb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    {\n",
      "  \"$id\": \"http://example.com/myURI.schema.json\",\n",
      "  \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n",
      "  \"additionalProperties\": false,\n",
      "  \"description\": \"Sample schema to help you get started.\",\n",
      "  \"properties\": {\n",
      "    \"brand\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"string\"\n",
      "    },\n",
      "    \"car_name\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"string\"\n",
      "    },\n",
      "    \"engine\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    },\n",
      "    \"fuel_type\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"string\"\n",
      "    },\n",
      "    \"km_driven\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    },\n",
      "    \"max_power\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    },\n",
      "    \"mileage\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    },\n",
      "    \"model\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"string\"\n",
      "    },\n",
      "    \"seats\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    },\n",
      "    \"seller_type\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"string\"\n",
      "    },\n",
      "    \"selling_price\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    },\n",
      "    \"transmission_type\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"string\"\n",
      "    },\n",
      "    \"vehicle_age\": {\n",
      "      \"description\": \"The type(v) type is used.\",\n",
      "      \"type\": \"number\"\n",
      "    }\n",
      "  },\n",
      "  \"title\": \"SampleRecord\",\n",
      "  \"type\": \"object\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(schema_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "038cf8fc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}