Readme updated

README.md

@@ -1,9 +1,9 @@
 # Interactive Classification for Deep Learning Interpretation
 
 We have designed and developed an interactive system that allows users to experiment with deep learning image classifiers and explore their robustness and sensitivity.
-Users are able to remove selected areas of an image in real time with classical computer vision inpainting algorithms, which allows users to ask a variety of "what if" questions by experimentally modifying images and seeing how the deep learning model reacts.
+Selected areas of an image can be removed in real time with classical computer vision inpainting algorithms, allowing users to ask a variety of "what if" questions by experimentally modifying images and seeing how the deep learning model reacts.
 The system also computes class activation maps for any selected class, which highlight the important semantic regions of an image the model uses for classification.
-The system runs fully in browser using Tensorflow.js, React, and SqueezeNet.
+The system runs fully in browser using Tensorflow.js, React, and SqueezeNet. An advanced inpainting version is also available using a server running the PatchMatch algorithm from the [GIMP Resynthesizer plugin](https://github.com/bootchk/resynthesizer).
 
 <!-- VIDEO LINK -->
 
@@ -33,6 +33,15 @@ yarn start
 
 ```
 
+## Advanced Inpainting
+
+The following steps are needed to set up PatchMatch inpainting:
+
+1. Clone the [Resynthesizer](https://github.com/bootchk/resynthesizer) repository and follow the instructions for building the project
+2. Find the `libresynthesizer.a` shared library in the `lib` folder and copy it to the `inpaint` folder in this repository
+3. Run `gcc resynth.c -L. -lresynthesizer -lm -lglib-2.0 -o prog` (may have to install glib2.0 first) to generate the prog executable
+4. You can now run `inpaint_server.py` and PatchMatch will be used as the inpainting algorithm.
+
 ## License
 
 MIT License. See [`LICENSE.md`](LICENSE.md).

inpaint/inpaint_server.py

@@ -8,10 +8,6 @@
 from flask import Flask, request, jsonify
 from flask_cors import CORS, cross_origin
 import json
-app = Flask(__name__)
-
-cors = CORS(app)
-app.config['CORS_HEADERS'] = 'Content-Type'
 
 def inpaint(img_arr, mask_arr):
 
@@ -35,14 +31,26 @@ def inpaint(img_arr, mask_arr):
     os.remove('.imgbytes')
     return nums
 
-@app.route('/')
-def hello_world():
-    return "Hello World!"
-
-@app.route('/inpaint', methods=['POST'])
-@cross_origin()
-def inpaint_req():
-    if request.method == 'POST':
-        data = request.get_json(force=True)
-        return jsonify(inpaint(data['image'], data['mask']))
-    return jsonify({"error": "Must be post request"})
+def create_app():
+    app = Flask(__name__)
+
+    cors = CORS(app)
+    app.config['CORS_HEADERS'] = 'Content-Type'
+
+    @app.route('/')
+    def hello_world():
+        return "Hello World!"
+
+    @app.route('/inpaint', methods=['POST'])
+    @cross_origin()
+    def inpaint_req():
+        if request.method == 'POST':
+            data = request.get_json(force=True)
+            return jsonify(inpaint(data['image'], data['mask']))
+        return jsonify({"error": "Must be post request"})
+
+    return app
+
+if __name__ == "__main__":
+    app = create_app()
+    app.run()

package.json

@@ -9,12 +9,11 @@
     "d3-scale-chromatic": "^1.2.0",
     "deeplearn": "^0.5.0",
     "deeplearn-squeezenet": "^0.2.0",
-    "loaders.css": "^0.1.2",
     "material-ui": "^0.20.0",
     "react": "^16.2.0",
     "react-dom": "^16.2.0",
-    "react-loaders": "^3.0.1",
     "react-scripts": "1.1.0",
+    "react-spinners": "^0.3.2",
     "requirejs": "^2.3.5",
     "stackblur-canvas": "^1.4.1"
   },

src/App.css

@@ -165,4 +165,13 @@ h3 {
 
 #loading-text {
   margin: 30px;
+}
+
+#inpaint-loader {
+  display: inline-block;
+  margin-left: 20px;
+}
+
+#modified-title {
+  margin-top: 0px;
 }

src/Modified.js

@@ -2,6 +2,7 @@ import React, { Component } from 'react';
 
 import {Table, TableHeader, TableHeaderColumn, TableBody, TableRow, Paper} from 'material-ui';
 import {canvasRGB} from 'stackblur-canvas';
+import {ClipLoader} from 'react-spinners';
 
 import {predict, inpaint, drawImage, createCompRows, drawCAM} from './util.js';
 import {IMAGENET_CLASSES} from './squeezenet/imagenet_classes.js';
@@ -73,7 +74,8 @@ class Modified extends Component {
 
     mouseUp = () => {
         this.setState({
-            mouseDown: false
+            mouseDown: false,
+            loading: true
         }) 
 
         inpaint(this.cImg.getContext('2d'), this.cDraw.getContext('2d')).then(img => {
@@ -88,7 +90,7 @@ class Modified extends Component {
                     activation: activation
                 });
             }.bind(this));
-         });
+         }).then(() => this.setState({ loading: false }));
     }
 
     mouseLeave = () => {
@@ -188,7 +190,7 @@ class Modified extends Component {
     render() {
         return (
             <div className="box" id="modified">
-                <Paper style={{marginBottom: 30, height: 227, width: 227}} zDepth={3}>
+                <Paper style={{marginBottom: 30, height: 227, width: 227, display: "inline-block"}} zDepth={3}>
                     <canvas id="modified-canvas" height="227px" width="227px" 
                             ref={cImg => this.cImg = cImg}> 
                     </canvas>
@@ -199,7 +201,10 @@ class Modified extends Component {
                             onMouseLeave={this.mouseLeave}>
                     </canvas>
                 </Paper>
-                <h3>Modified Image</h3>
+                <div id="inpaint-loader">
+                    <ClipLoader id="inpaint-loader" color="rgb(63, 81, 181)" loading={this.state.loading} />
+                </div>
+                <h3 id="modified-title">Modified Image</h3>
                 <Table className="table" onRowSelection={this.drawCAM}>
                     <TableHeader adjustForCheckbox={false} displaySelectAll={false}>
                         <TableRow className="header-row" onCellClick={(e, f, g) => this.orderChanged(e, f, g)}>

src/Options.js

@@ -12,7 +12,7 @@ class Options extends Component {
     this.state = {
       tSlide: 0,
       slides: [
-        <span><b>Interactive Classificati on</b> allows you to explore how computers see by modifying images.</span>,
+        <span><b>Interactive Classification</b> allows you to explore how computers see by modifying images.</span>,
         <span>The <b>Class</b> column tells you what the computer thinks the image is, and the <b>Confidence %</b> column tells you how confident it is in its choice.</span>,
         <span>You can click on a row to see the <b>Class Activation Map</b>. This is a heatmap showing which areas of the image the computer found most important when choosing that class.</span>,
         <span>Hover over the <b>Modified Image</b> to see a yellow circle. Draw, by clicking and dragging over the image, to remove an object.</span>,