Biswarup Banerjee gsoc pre reqs

README.md

@@ -0,0 +1,58 @@
+# Spark UI by @techguybiswa
+
+![Github Actions Status](jupyter/workflows/Build/badge.svg)
+
+playing around with jupyter
+
+## IMP NOTE: 
+
+After you start multiple spark clusters on localhost:4040, 4041 and so on , make sure to  install <a href="https://chrome.google.com/webstore/detail/moesif-orign-cors-changer/digfbfaphojjndkpccljibejjbppifbc"> Moesif CORS </a> so as to disable CORS. Or else none of the APIs that connect the extension with the spark UI  will work.
+
+## Requirements
+
+* JupyterLab >= 1.0
+
+## Install
+
+```bash
+jupyter labextension install bisso
+```
+
+## Contributing
+
+### Install
+
+The `jlpm` command is JupyterLab's pinned version of
+[yarn](https://yarnpkg.com/) that is installed with JupyterLab. You may use
+`yarn` or `npm` in lieu of `jlpm` below.
+
+```bash
+# Clone the repo to your local environment
+# Move to bisso directory
+# Install dependencies
+jlpm
+# Build Typescript source
+jlpm build
+# Link your development version of the extension with JupyterLab
+jupyter labextension link .
+# Rebuild Typescript source after making changes
+jlpm build
+# Rebuild JupyterLab after making any changes
+jupyter lab build
+```
+
+You can watch the source directory and run JupyterLab in watch mode to watch for changes in the extension's source and automatically rebuild the extension and application.
+
+```bash
+# Watch the source directory in another terminal tab
+jlpm watch
+# Run jupyterlab in watch mode in one terminal tab
+jupyter lab --watch
+```
+
+### Uninstall
+
+```bash
+jupyter labextension uninstall bisso
+```
+

lib/index.d.ts

@@ -0,0 +1,7 @@
+import { JupyterFrontEndPlugin } from "@jupyterlab/application";
+import "../style/index.css";
+/**
+ * Initialization data for the spark_ui_tab extension.
+ */
+declare const extension: JupyterFrontEndPlugin<void>;
+export default extension;

lib/index.js

@@ -0,0 +1,160 @@
+import { ICommandPalette, MainAreaWidget } from "@jupyterlab/apputils";
+import { Widget } from "@lumino/widgets";
+import "../style/index.css";
+function activate(app, palette) {
+    console.log("In activate");
+    /* Create main view components */
+    const sparkWidget = new Widget();
+    sparkWidget.title.label = "Open Spark UI";
+    const widget = new MainAreaWidget({ content: sparkWidget });
+    widget.id = "spark-ui";
+    widget.title.label = "Spark UI";
+    widget.title.closable = true;
+    const heading = document.createElement("h1");
+    heading.innerHTML = "Spark Interface by @techguybiswa";
+    heading.style.marginLeft = "5%";
+    const sparkContainer = document.createElement("div");
+    sparkContainer.style.textAlign = "center";
+    sparkContainer.style.height = "70vh";
+    let fetchDataOfPySparkFromDropdown = (event) => {
+        console.log(event.target.value);
+        document.getElementById("portnumber").value = event.target.value;
+        fetchDataOfPySpark();
+    };
+    let fetchDataOfPySpark = async () => {
+        let portNumber = document.getElementById("portnumber")
+            .value;
+        try {
+            document.getElementById("applicationRes").innerHTML = `<p>Loading application data...</p>`;
+            document.getElementById("executorRes").innerHTML = `<p>Loading executors data...</p>`;
+            document.getElementById("environmentRes").innerHTML = `<p>Loading environment data...</p>`;
+            let applicationResponse = await fetch(`http://localhost:${portNumber}/api/v1/applications/`);
+            let applicationResult = await applicationResponse.json();
+            console.log("result is ", applicationResult);
+            document.getElementById("applicationRes").innerHTML = `
+      <p>Applications Detail</p>
+      <p>Number of applications running : ${applicationResult.length}</p>
+        <ul>
+        <li>App Id: ${applicationResult[0].id}</li>
+        <li>App Name: ${applicationResult[0].name}</li>
+        </ul>
+      `;
+            let appId = applicationResult[0].id;
+            let executorsResponse = await fetch(`http://localhost:${portNumber}/api/v1/applications/${appId}/executors`);
+            let executors = await executorsResponse.json();
+            console.log("executors", executors);
+            document.getElementById("executorRes").innerHTML = `
+      <p>Executors Detail</p>
+      <ul>
+      <li>Executor Id: ${executors[0].id}</li>
+      <li>Host Port: ${executors[0].hostPort}</li>
+      <li>Add Time: ${executors[0].addTime}</li>
+      <li>Max Memory: ${executors[0].maxMemory}</li>
+      </ul>
+    `;
+            let environmentResponse = await fetch(`http://localhost:${portNumber}/api/v1/applications/${appId}/environment`);
+            let environment = await environmentResponse.json();
+            console.log("environment", environment);
+            document.getElementById("environmentRes").innerHTML = `
+      <p>Environment Detail</p>
+
+      <ul>
+      <li>Java verion : ${environment.runtime.javaVersion}</li>
+      <li>Java Home: ${environment.runtime.javaHome}</li>
+      <li>Scala Version: ${environment.runtime.scalaVersion}</li>
+      </ul>
+    `;
+        }
+        catch (err) {
+            document.getElementById("applicationRes").innerHTML = `<p>Error while loading application data.</p>`;
+            document.getElementById("executorRes").innerHTML = `<p>Error while loading executors data.</p>`;
+            document.getElementById("environmentRes").innerHTML = `<p>Error while loading environment data.</p>`;
+            console.log("There was an error");
+        }
+    };
+    let listOfAllPorts = [];
+    let fetchListOfAllPorts = async (initialPort) => {
+        try {
+            document.getElementById("getPorts").disabled = true;
+            document.getElementById("getPorts").innerHTML = `Fetching all active ports...`;
+            let apiRes = await fetch(`http://localhost:${initialPort}/api/v1/applications/`);
+            if (apiRes) {
+                listOfAllPorts = [...listOfAllPorts, initialPort];
+            }
+            fetchListOfAllPorts(initialPort + 1);
+            //recursively call the function fetchListOfAllPorts() to check which all ports are active
+            // the end condition of the recursion is when the port is not available 
+        }
+        catch (err) {
+            document.getElementById("selectPort").innerHTML = "";
+            document.getElementById("getPorts").disabled = false;
+            document.getElementById("getPorts").innerHTML = `Fetch all active ports`;
+            let selectElement = document.getElementById("selectPort");
+            listOfAllPorts.map(eachPort => {
+                selectElement.add(new Option(`Port ${eachPort}`, eachPort));
+            });
+            listOfAllPorts = [];
+            console.log(listOfAllPorts);
+        }
+    };
+    /* Create input field */
+    const submitButton = document.createElement("button");
+    submitButton.id = "submitBtn";
+    submitButton.value = "4040";
+    submitButton.innerHTML = "FETCH";
+    submitButton.onclick = () => fetchDataOfPySpark();
+    const getPortsButton = document.createElement("button");
+    getPortsButton.id = "getPorts";
+    getPortsButton.innerHTML = "Fetch all active ports";
+    getPortsButton.onclick = () => fetchListOfAllPorts(4040);
+    const portInput = document.createElement("div");
+    portInput.innerHTML = `<div class="wrapper">
+      <p>Show details of port</p>
+      <input class="input" id="portnumber" placeholder="4040" type="text" value="4040">
+    </div>`;
+    const select = document.createElement("select");
+    select.id = "selectPort";
+    select.onchange = () => fetchDataOfPySparkFromDropdown(event);
+    const applicationResponse = document.createElement("div");
+    applicationResponse.setAttribute("id", "applicationRes");
+    const executorsResponse = document.createElement("div");
+    executorsResponse.setAttribute("id", "executorRes");
+    const environmentResponse = document.createElement("div");
+    environmentResponse.setAttribute("id", "environmentRes");
+    sparkWidget.node.appendChild(heading);
+    sparkWidget.node.appendChild(portInput);
+    sparkWidget.node.appendChild(submitButton);
+    sparkWidget.node.appendChild(getPortsButton);
+    sparkWidget.node.appendChild(select);
+    sparkWidget.node.appendChild(applicationResponse);
+    sparkWidget.node.appendChild(executorsResponse);
+    sparkWidget.node.appendChild(environmentResponse);
+    sparkWidget.node.appendChild(sparkContainer);
+    /* Add an application command */
+    const command = "spark:open";
+    app.commands.addCommand(command, {
+        label: "Spark UI",
+        execute: () => {
+            if (!widget.isAttached) {
+                // Attach the widget to the main work area if it's not there
+                app.shell.add(widget, "main");
+            }
+            // Refresh the picture in the widget
+            sparkWidget.update();
+            // Activate the widget
+            app.shell.activateById(widget.id);
+        }
+    });
+    /* Add the command to the palette. */
+    palette.addItem({ command, category: "Spark" });
+}
+/**
+ * Initialization data for the spark_ui_tab extension.
+ */
+const extension = {
+    id: "spark_ui",
+    autoStart: true,
+    requires: [ICommandPalette],
+    activate: activate
+};
+export default extension;

package.json

@@ -0,0 +1,49 @@
+{
+  "name": "bisso",
+  "version": "0.1.0",
+  "description": "playing around with jupyter",
+  "keywords": [
+    "jupyter",
+    "jupyterlab",
+    "jupyterlab-extension"
+  ],
+  "homepage": "jupyter",
+  "bugs": {
+    "url": "jupyter/issues"
+  },
+  "license": "BSD-3-Clause",
+  "author": "",
+  "files": [
+    "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}",
+    "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}"
+  ],
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "style": "style/index.css",
+  "repository": {
+    "type": "git",
+    "url": "jupyter.git"
+  },
+  "scripts": {
+    "build": "tsc",
+    "clean": "rimraf lib tsconfig.tsbuildinfo",
+    "prepare": "jlpm run clean && jlpm run build",
+    "watch": "tsc -w"
+  },
+  "dependencies": {
+    "@jupyterlab/application": "^2.0.2",
+    "@jupyterlab/apputils": "^2.0.2",
+    "@lumino/messaging": "^1.3.3",
+    "@lumino/widgets": "^1.11.1"
+  },
+  "devDependencies": {
+    "rimraf": "^2.6.1",
+    "typescript": "~3.7.0"
+  },
+  "sideEffects": [
+    "style/*.css"
+  ],
+  "jupyterlab": {
+    "extension": true
+  }
+}