01_d_SplitDataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import src.data.Dataset as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_CSV = os.path.abspath(\n",
    "    \"./data/raw/emptyNonEmptyDataset_ETL/28570Images_binary.csv\"\n",
    ")\n",
    "OUTPUT_FOLDER = os.path.abspath(\"./data/raw/emptyNonEmptyDataset_ETL\")\n",
    "\n",
    "print(f\"DATASET_CSV:     {DATASET_CSV}\")\n",
    "print(f\"OUTPUT_FOLDER:   {OUTPUT_FOLDER}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dt.load_from_csv(DATASET_CSV)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "percentaje_train = 0.70\n",
    "percentaje_val = 0.15\n",
    "percentaje_test = 0.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_images = len(dataset[\"file_name\"])\n",
    "\n",
    "train_file_name = os.path.join(OUTPUT_FOLDER, str(number_of_images) + \"Train.csv\")\n",
    "val_file_name = os.path.join(OUTPUT_FOLDER, str(number_of_images) + \"Validation.csv\")\n",
    "test_file_name = os.path.join(OUTPUT_FOLDER, str(number_of_images) + \"Test.csv\")\n",
    "\n",
    "print(f\"train_file_name:   {train_file_name}\")\n",
    "print(f\"val_val_file_name: {val_file_name}\")\n",
    "print(f\"test_file_name:    {test_file_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset, val_dataset, test_dataset = dt.split_dataset(\n",
    "    dataset, percentaje_train, percentaje_val, percentaje_test\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt.dataset_to_csv(train_dataset, train_file_name)\n",
    "dt.dataset_to_csv(val_dataset, val_file_name)\n",
    "dt.dataset_to_csv(test_dataset, test_file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    \"==================================================================================\"\n",
    ")\n",
    "print(f\"Calculos para TRAIN: {percentaje_train*100}%:\")\n",
    "print(\n",
    "    \"==================================================================================\"\n",
    ")\n",
    "print(\n",
    "    f\"- Valor teorico: {len(dataset)} x {percentaje_train*100}% = {len(dataset)*percentaje_train}\"\n",
    ")\n",
    "print(f\"- Valor generado: {len(train_dataset)}\")\n",
    "print()\n",
    "print(\n",
    "    f\"- Valor teorico VACIA: {len(dataset[dataset['binary_label'] == 0])} x {percentaje_train*100}% = {len(dataset[dataset['binary_label'] == 0])*percentaje_train}\"\n",
    ")\n",
    "print(\n",
    "    f\"- Valor generado VACIA: {len(train_dataset[train_dataset['binary_label'] == 0])}\"\n",
    ")\n",
    "print()\n",
    "print(\n",
    "    f\"- Valor teorico ANIMAL: {len(dataset[dataset['binary_label'] == 1])} x {percentaje_train*100}% = {len(dataset[dataset['binary_label'] == 1])*percentaje_train}\"\n",
    ")\n",
    "print(\n",
    "    f\"- Valor generado ANIMAL: {len(train_dataset[train_dataset['binary_label'] == 1])}\"\n",
    ")\n",
    "print()\n",
    "print(\n",
    "    \"==================================================================================\"\n",
    ")\n",
    "print(f\"Calculos para VALIDATION: {percentaje_val*100}%:\")\n",
    "print(\n",
    "    \"==================================================================================\"\n",
    ")\n",
    "print(\n",
    "    f\"- Valor teorico: {len(dataset)} x {percentaje_val*100}% = {len(dataset)*percentaje_val}\"\n",
    ")\n",
    "print(f\"- Valor generado: {len(val_dataset)}\")\n",
    "print()\n",
    "print(\n",
    "    f\"- Valor teorico VACIA: {len(dataset[dataset['binary_label'] == 0])} x {percentaje_val*100}% = {len(dataset[dataset['binary_label'] == 0])*percentaje_val}\"\n",
    ")\n",
    "print(f\"- Valor generado VACIA: {len(val_dataset[val_dataset['binary_label'] == 0])}\")\n",
    "print()\n",
    "print(\n",
    "    f\"- Valor teorico ANIMAL: {len(dataset[dataset['binary_label'] == 1])} x {percentaje_val*100}% = {len(dataset[dataset['binary_label'] == 1])*percentaje_val}\"\n",
    ")\n",
    "print(f\"- Valor generado ANIMAL: {len(val_dataset[val_dataset['binary_label'] == 1])}\")\n",
    "print()\n",
    "print(\n",
    "    \"==================================================================================\"\n",
    ")\n",
    "print(f\"Calculos para TEST: {percentaje_test*100}%:\")\n",
    "print(\n",
    "    \"==================================================================================\"\n",
    ")\n",
    "print(\n",
    "    f\"- Valor teorico: {len(dataset)} x {percentaje_test*100}% = {len(dataset)*percentaje_test}\"\n",
    ")\n",
    "print(f\"- Valor generado: {len(test_dataset)}\")\n",
    "print()\n",
    "print(\n",
    "    f\"- Valor teorico VACIA: {len(dataset[dataset['binary_label'] == 0])} x {percentaje_test*100}% = {len(dataset[dataset['binary_label'] == 0])*percentaje_test}\"\n",
    ")\n",
    "print(f\"- Valor generado VACIA: {len(test_dataset[test_dataset['binary_label'] == 0])}\")\n",
    "print()\n",
    "print(\n",
    "    f\"- Valor teorico ANIMAL: {len(dataset[dataset['binary_label'] == 1])} x {percentaje_test*100}% = {len(dataset[dataset['binary_label'] == 1])*percentaje_test}\"\n",
    ")\n",
    "print(\n",
    "    f\"- Valor generado ANIMAL: {len(test_dataset[test_dataset['binary_label'] == 1])}\"\n",
    ")\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "ruby"
    }
   },
   "outputs": [],
   "source": [
    "# Añadir una columna 'subset' al dataset original\n",
    "dataset['subset'] = 'none'\n",
    "\n",
    "# Asignar el valor correspondiente a cada subconjunto\n",
    "dataset.loc[dataset['file_name'].isin(train_dataset['file_name']), 'subset'] = 'train'\n",
    "dataset.loc[dataset['file_name'].isin(val_dataset['file_name']), 'subset'] = 'validation'\n",
    "dataset.loc[dataset['file_name'].isin(test_dataset['file_name']), 'subset'] = 'test'\n",
    "\n",
    "# Verificar los cambios\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt.dataset_to_csv(\n",
    "    dataset, os.path.join(OUTPUT_FOLDER, str(number_of_images) + \"Images_subset.csv\")\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "megaclassifier",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}