Revert "Revert "Fix 159 uwsgi timeout""

Dockerfile-spfy

@@ -11,8 +11,6 @@ FROM superphy/docker-flask-conda:2.0.0
 COPY nginx.conf /etc/nginx/conf.d/
 # Copy the base uWSGI ini file to enable default dynamic uwsgi process number
 COPY uwsgi-base.ini /etc/uwsgi/
-# Add maximum upload of 100 m
-COPY upload_60G.conf /etc/nginx/conf.d/
 #TEMPORARY
 RUN rm /etc/nginx/conf.d/upload_100m.conf
 

app/config.py

@@ -1,5 +1,5 @@
 #for secret key, do a os.urandom(24).encode('hex')
-SECRET_KEY = 'is-that-daisy'
+SECRET_KEY = '77cc1b212cc8e41be655902b0a4cca30806db5db88299e41'
 REDIS_URL = 'redis://redis:6379/0'
 # actual queues aren't defined here, they are in their respective supervisord.conf worker call
 # this is done to isolate the RQ-Blazegraph worker to avoid race conditions
@@ -43,7 +43,7 @@
 #database defaults
 DATABASE_ENABLED = True
 database = {}
-database['blazegraph_url'] = 'http://blazegraph:8080/bigdata/sparql'
+database['blazegraph_url'] = 'http://192.168.0.1:9999/blazegraph/sparql'
 #database['blazegraph_url'] = 'http://localhost:9000/blazegraph/namespace/superphy/sparql'
 #### end of savvy.py stuff
 
@@ -52,4 +52,4 @@
 # If you're using Sentry to collect your runtime exceptions, you can use this
 # to configure RQ for it in a single step
 # NOTE!!!: There is a bug with Raven that needs to be accounted for in RQ config. You must prefix your sentry dsn with sync+ eg 'sync+https://...' see https://github.com/nvie/rq/issues/350 . As of Mar.'17 this hasn't been fixed.
-#SENTRY_DSN = 'sync+https://public:secret@example.com/1'
+SENTRY_DSN = 'sync+https://508d71e54ad84c9483320f051cc798ce:3efb1bc16af34fe8b3e5861382a83c9c@sentry.io/135389'

app/routes/ra_posts.py

@@ -162,6 +162,7 @@ def handle_singleton(jobs_dict):
 # this is a modification of the old upload() methods in views.py
 @bp_ra_posts.route('/api/v0/upload', methods=['POST'])
 def upload():
+    print 'upload(): received req. at ' + str(datetime.now().strftime("%Y-%m-%d-%H-%M"))
     recaptcha = ReCaptcha(app=current_app)
     if recaptcha.verify():
         form = request.form

app/routes/views.py

@@ -36,6 +36,9 @@ def upload():
         options['serotype']=True
         options['pi']=90
 
+	# for compat w 4.3.3
+	options['bulk'] = False
+
         # processing form data
         for key, value in form.items():
             #we need to convert lower-case true/false in js to upper case in python

app/static/js/main.js

@@ -96,6 +96,7 @@ app.controller('SpfyController', [
             fd.append('options.serotype', $scope.formData.options.serotype);
             fd.append('options.pi', $scope.pi);
             fd.append('g-recaptcha-response', $scope.response);
+	    fd.append('options.bulk', false);
             $log.log($scope.response);
             $log.log($scope.formData);
             $scope.loading = true;

app/uwsgi.ini

@@ -2,3 +2,13 @@
 [uwsgi]
 wsgi-file = main.py
 callable = app
+socket = /tmp/uwsgi.sock
+chown-socket = nginx:nginx
+chmod-socket = 664
+cheaper = 2
+processes = 16
+socket-timeout = 1800
+chunked-input-timeout = 1800
+http-timeout = 1800
+harakiri = 1800
+post-buffering = 1

docker-compose.yml

@@ -6,22 +6,25 @@ services:
       dockerfile: Dockerfile-spfy
     image: backend
     ports:
-    - "8000:80"
+    - "8090:80"
     depends_on:
     - redis
-    - blazegraph
     volumes:
     - /datastore
+    networks:
+      - dockernet
 
   reactapp:
     build:
       context: .
       dockerfile: Dockerfile-reactapp
     image: reactapp
     ports:
-    - "8090:5000"
+    - "8091:5000"
     depends_on:
     - webserver
+    networks:
+      - dockernet
 
   worker:
     build:
@@ -34,6 +37,8 @@ services:
     - webserver
     depends_on:
     - webserver
+    networks:
+      - dockernet
 
   worker-blazegraph-ids:
     build:
@@ -44,6 +49,8 @@ services:
     - webserver
     depends_on:
     - webserver
+    networks:
+      - dockernet
 
   worker-priority:
     build:
@@ -54,16 +61,21 @@ services:
     - webserver
     depends_on:
     - webserver
+    networks:
+      - dockernet
 
   redis:
     image: redis:3.2
     command: redis-server --appendonly yes # for persistance
     volumes:
     - /data
+    networks:
+      - dockernet
 
-  blazegraph:
-    image: superphy/blazegraph:2.1.4-inferencing
-    ports:
-    - "8080:8080"
-    volumes:
-    - /var/lib/jetty/
+networks:
+  dockernet:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 192.168.0.0/24
+          gateway: 192.168.0.1

nginx.conf

@@ -5,16 +5,23 @@ server {
   gzip_min_length  1000;
   gzip_proxied     expired no-cache no-store private auth;
   gzip_types       text/plain application/x-javascript text/xml text/css application/xml multipart/form-data;
-
-  uwsgi_connect_timeout       1800;
-  uwsgi_send_timeout          1800;
-  uwsgi_read_timeout          1800;
-  send_timeout                1800;
 
   location / {
       try_files $uri @app;
   }
   location @app {
+      uwsgi_connect_timeout       1800;
+      uwsgi_send_timeout          1800;
+      uwsgi_read_timeout          1800;
+      send_timeout                1800;
+
+      client_body_in_file_only clean;
+      client_body_buffer_size 256K;
+      client_max_body_size 60g;
+
+      proxy_read_timeout 1200;
+      keepalive_timeout 30;
+
       include uwsgi_params;
       uwsgi_pass unix:///tmp/uwsgi.sock;
   }

uwsgi-base.ini

@@ -5,5 +5,9 @@ chmod-socket = 664
 cheaper = 2
 processes = 16
 socket-timeout = 1800
+chunked-input-timeout = 1800
+http-timeout = 1800
 harakiri = 1800
-post-buffering = 1024
+post-buffering = 1
+req-logger = file:/tmp/reqlog
+logger = file:/tmp/errlog