Spark ML Logistic Regression.ipynb

{
    "nbformat_minor": 1, 
    "cells": [
        {
            "execution_count": 1, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "Waiting for a Spark session to start...\nSpark Initialization Done! ApplicationId = app-20190903091321-0003\nKERNEL_ID = f1ba51b4-1b5b-47c8-abb0-07567c8b35db\n"
                }
            ], 
            "source": "# The code was removed by Watson Studio for sharing."
        }, 
        {
            "execution_count": 2, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "splits = df.randomSplit([0.8, 0.2])\ndf_train = splits[0]\ndf_test = splits[1]"
        }, 
        {
            "execution_count": 3, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml.feature import StringIndexer, OneHotEncoder\nfrom pyspark.ml.linalg import Vectors\nfrom pyspark.ml.feature import VectorAssembler\nfrom pyspark.ml.feature import Normalizer\n\n\nindexer = StringIndexer(inputCol=\"class\", outputCol=\"label\")\nvectorAssembler = VectorAssembler(inputCols=[\"x\",\"y\",\"z\"],outputCol=\"features\")\nnormalizer = Normalizer(inputCol=\"features\", outputCol=\"features_norm\", p=1.0)"
        }, 
        {
            "execution_count": 4, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml.classification import LogisticRegression\n\nlr = LogisticRegression(maxIter=10, regParam=0.3, elasticNetParam=0.8)"
        }, 
        {
            "execution_count": 5, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "from pyspark.ml import Pipeline\npipeline = Pipeline(stages=[indexer, vectorAssembler, normalizer,lr])"
        }, 
        {
            "execution_count": 6, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "model = pipeline.fit(df_train)"
        }, 
        {
            "execution_count": 7, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "prediction = model.transform(df_train)"
        }, 
        {
            "execution_count": 8, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "output_type": "stream", 
                    "name": "stdout", 
                    "text": "root\n |-- x: integer (nullable = true)\n |-- y: integer (nullable = true)\n |-- z: integer (nullable = true)\n |-- source: string (nullable = true)\n |-- class: string (nullable = true)\n |-- label: double (nullable = false)\n |-- features: vector (nullable = true)\n |-- features_norm: vector (nullable = true)\n |-- rawPrediction: vector (nullable = true)\n |-- probability: vector (nullable = true)\n |-- prediction: double (nullable = false)\n\n"
                }
            ], 
            "source": "prediction.printSchema()"
        }, 
        {
            "execution_count": 9, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 9, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "0.1293416049064505"
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\nbinEval = MulticlassClassificationEvaluator().setMetricName(\"accuracy\") .setPredictionCol(\"prediction\").setLabelCol(\"label\")\nbinEval.evaluate(prediction) "
        }, 
        {
            "execution_count": 10, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [], 
            "source": "prediction = model.transform(df_test)"
        }, 
        {
            "execution_count": 11, 
            "cell_type": "code", 
            "metadata": {}, 
            "outputs": [
                {
                    "execution_count": 11, 
                    "metadata": {}, 
                    "data": {
                        "text/plain": "0.12903906777752758"
                    }, 
                    "output_type": "execute_result"
                }
            ], 
            "source": "binEval.evaluate(prediction) "
        }
    ], 
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3.6 with Spark", 
            "name": "python36", 
            "language": "python3"
        }, 
        "language_info": {
            "mimetype": "text/x-python", 
            "nbconvert_exporter": "python", 
            "version": "3.6.8", 
            "name": "python", 
            "file_extension": ".py", 
            "pygments_lexer": "ipython3", 
            "codemirror_mode": {
                "version": 3, 
                "name": "ipython"
            }
        }
    }, 
    "nbformat": 4
}