Ensemble Learning - Classification Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f447ba47-9654-40c7-b29f-168e7cbd97a9",
   "metadata": {},
   "source": [
    "# Breast cancer Dataset from sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9993b5d6-449f-4cac-8edb-8dd3e22a4459",
   "metadata": {},
   "source": [
    "### Import necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22c7f10f-bdff-4b7a-ad73-24cbccd23069",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6268473-8792-4488-b24c-f6d7215977cf",
   "metadata": {},
   "source": [
    "### Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "24d58e4c-af52-439f-82d7-b32f8e70eef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "breast_cancer_data = load_breast_cancer()\n",
    "X = breast_cancer_data.data\n",
    "y = breast_cancer_data.target\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53bf333e-5302-49fd-a9ed-9075a63497cb",
   "metadata": {},
   "source": [
    "### Splitting data into training and testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "873781e7-ce74-40bd-884d-0706bc8c4a2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X,y,random_state= 42,test_size=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eddbefbc-4270-4f43-a1f1-725198c2f36f",
   "metadata": {},
   "source": [
    "### Training the RandomForestClassifier(Bagging)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bc8724f8-98fa-4044-b101-f90a67990d15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 0 1 1 0 0 0 0 1 1 0 1 0 1 0 1 1 1 0 1 1 0 1 1 1 1 1 1 0 1 1 1 1 1 1 0\n",
      " 1 0 1 1 0 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 0 0 1 1 0 0 1 1 1 0 0 1 1 0 0 1 0\n",
      " 1 1 1 1 1 1 0 1 1 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 1 0 0 1 0 0 1 1 1 0 1 1 0\n",
      " 1 1 0 1 0 1 1 1 0 1 1 1 0 1 0 0 1 1 0 0 0 1 1 1 0 1 1 1 0 1 0 1 1 0 1 0 0\n",
      " 0 1 0 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 1]\n",
      "Accuracy Results using different metrics\n",
      "Accuracy_score: 97.08%\n",
      "Precision_score: 96.40%\n",
      "Recall_score: 99.07%\n",
      "F1_score: 97.72%\n"
     ]
    }
   ],
   "source": [
    "# Initialize RandomForestClassifier\n",
    "rf = RandomForestClassifier(n_estimators=100,random_state =42)\n",
    "\n",
    "# Train the model\n",
    "rf.fit(X_train,y_train)\n",
    "\n",
    "#Make predictions on the test set\n",
    "predictions = rf.predict(X_test)\n",
    "\n",
    "#Evaluation of Accuracy\n",
    "from sklearn.metrics import precision_score,recall_score,f1_score,accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(y_test,predictions)*100 \n",
    "precision = precision_score(y_test,predictions)*100 \n",
    "recall = recall_score(y_test,predictions)*100 \n",
    "f1 = f1_score(y_test,predictions)*100 \n",
    "\n",
    "print(\"Accuracy Results using different metrics\")\n",
    "print(f\"Accuracy_score: {accuracy:.2f}%\")\n",
    "\n",
    "# Measure out of the predicted positves how many are actually positive.\n",
    "print(f\"Precision_score: {precision:.2f}%\")\n",
    "\n",
    "# Measures out of the total actual positives how many are true positives.\n",
    "print(f\"Recall_score: {recall:.2f}%\")\n",
    "\n",
    "# Combines recall and precision\n",
    "print(f\"F1_score: {f1:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6302553-7ccd-4070-870d-45866beb6f47",
   "metadata": {},
   "source": [
    "### Boosting Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f1966441-dc4b-47e3-8704-97d59a975bac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bree/myenv/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Results using different metrics\n",
      "Accuracy_score: 97.66%\n",
      "Precision_score: 98.15%\n",
      "Recall_score: 98.15%\n",
      "F1_score: 98.15%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# Initialize the base model(weak learner)\n",
    "base_model = DecisionTreeClassifier(max_depth = 1)\n",
    "\n",
    "# Initialize AdaBoostClassifier\n",
    "ada_clf = AdaBoostClassifier(estimator=base_model, n_estimators=50, random_state=42)\n",
    "\n",
    "# Train the AdaBoost model\n",
    "ada_clf.fit(X_train,y_train)\n",
    "\n",
    "# Make predictions\n",
    "predictions = ada_clf.predict(X_test)\n",
    "\n",
    "#Evaluation of Accuracy\n",
    "from sklearn.metrics import precision_score,recall_score,f1_score,accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(y_test,predictions)*100 \n",
    "precision = precision_score(y_test,predictions)*100 \n",
    "recall = recall_score(y_test,predictions)*100 \n",
    "f1 = f1_score(y_test,predictions)*100 \n",
    "\n",
    "print(\"Accuracy Results using different metrics\")\n",
    "print(f\"Accuracy_score: {accuracy:.2f}%\")\n",
    "\n",
    "# Measure out of the predicted positves how many are actually positive.\n",
    "print(f\"Precision_score: {precision:.2f}%\")\n",
    "\n",
    "# Measures out of the total actual positives how many are true positives.\n",
    "print(f\"Recall_score: {recall:.2f}%\")\n",
    "\n",
    "# Combines recall and precision\n",
    "print(f\"F1_score: {f1:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1a7a5f7-e37a-42aa-b44e-cf232615ffaa",
   "metadata": {},
   "source": [
    "### Stacking Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b023a9de-e9e3-45da-ae27-4db541807ee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import StackingClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# Initialize base models\n",
    "base_models = [\n",
    "    ('dt', DecisionTreeClassifier(max_depth=3)),  # Decision Tree\n",
    "    ('svc', SVC(kernel='linear', probability=True))  # Support Vector Classifier\n",
    "]\n",
    "\n",
    "# Initialize meta-learner (Logistic Regression)\n",
    "meta_model = LogisticRegression()\n",
    "\n",
    "# Create StackingClassifier\n",
    "stack_clf = StackingClassifier(estimators=base_models, final_estimator=meta_model, cv=5)\n",
    "\n",
    "# Train the stacking model\n",
    "stack_clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred_stack = stack_clf.predict(X_test)\n",
    "\n",
    "#Evaluation of Accuracy\n",
    "from sklearn.metrics import precision_score,recall_score,f1_score,accuracy_score\n",
    "\n",
    "accuracy = accuracy_score(y_test,predictions)*100 \n",
    "precision = precision_score(y_test,predictions)*100 \n",
    "recall = recall_score(y_test,predictions)*100 \n",
    "f1 = f1_score(y_test,predictions)*100 \n",
    "\n",
    "print(\"Accuracy Results using different metrics\")\n",
    "print(f\"Accuracy_score: {accuracy:.2f}%\")\n",
    "\n",
    "# Measure out of the predicted positves how many are actually positive.\n",
    "print(f\"Precision_score: {precision:.2f}%\")\n",
    "\n",
    "# Measures out of the total actual positives how many are true positives.\n",
    "print(f\"Recall_score: {recall:.2f}%\")\n",
    "\n",
    "# Combines recall and precision\n",
    "print(f\"F1_score: {f1:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03ad4147-8a19-4373-bc6b-9b867f30fd2e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}