Merge pull request #441 from pierotofy/hang

Hang

.env

@@ -5,5 +5,5 @@ WO_SSL=NO
 WO_SSL_KEY=
 WO_SSL_CERT=
 WO_SSL_INSECURE_PORT_REDIRECT=80
-WO_DEBUG=YES
+WO_DEBUG=NO
 WO_BROKER=redis://broker

README.md

@@ -10,7 +10,6 @@ A free, user-friendly, extendable application and [API](http://docs.webodm.org)
 
 * [Getting Started](#getting-started)
     * [Add More Processing Nodes](#add-more-processing-nodes)
-    * [Security](#security)
     * [Enable SSL](#enable-ssl)
     * [Where Are My Files Stored?](#where-are-my-files-stored)
     * [Common Troubleshooting](#common-troubleshooting)
@@ -84,16 +83,6 @@ Adding more processing nodes will allow you to run multiple jobs in parallel.
 
 You **will not be able to distribute a single job across multiple processing nodes**. We are actively working to bring this feature to reality, but we're not there yet. 
 
-### Security
-
-If you want to run WebODM in production, make sure to pass the `--no-debug` flag while starting WebODM:
-
-```bash
-./webodm.sh restart --no-debug
-```
-
-This will disable the `DEBUG` flag from `webodm/settings.py` within the docker container. This is [really important](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-DEBUG).
-
 ### Enable SSL
 
 WebODM has the ability to automatically request and install a SSL certificate via [Let’s Encrypt](https://letsencrypt.org/), or you can manually specify your own key/certificate pair.

app/static/app/js/components/TaskListItem.jsx

@@ -500,7 +500,6 @@ class TaskListItem extends React.Component {
                   "Process exited with code 1" means that part of the processing failed. Try tweaking the <a href="javascript:void(0);" onClick={this.startEditing}>Task Options</a> as follows:
                   <ul>
                     <li>Increase the <b>min-num-features</b> option, especially if your images have lots of vegetation</li>
-                    <li>Enable the <b>use-pmvs</b> option.</li>
                   </ul>
                   Still not working? Upload your images somewhere like <a href="https://www.dropbox.com/" target="_blank">Dropbox</a> or <a href="https://drive.google.com/drive/u/0/" target="_blank">Google Drive</a> and <a href="http://community.opendronemap.org/c/webodm" target="_blank">open a topic</a> on our community forum, making 
                   sure to include a <a href="javascript:void(0);" onClick={this.downloadTaskOutput}>copy of your task's output</a> (the one you see above <i className="fa fa-arrow-up"></i>, click to <a href="javascript:void(0);" onClick={this.downloadTaskOutput}>download</a> it). Our awesome contributors will try to help you! <i className="fa fa-smile-o"></i>

devenv.sh

@@ -4,6 +4,8 @@ __dirname=$(cd $(dirname "$0"); pwd -P)
 
 ${__dirname}/webodm.sh checkenv
 
+export WO_DEBUG=YES
+
 usage(){
   echo "Usage: $0 <command> [options]"
   echo

docker-compose.nodeodm.yml

@@ -13,3 +13,5 @@ services:
     container_name: node-odm-1
     ports:
       - "3000"
+    restart: on-failure:10
+    oom_score_adj: 500

docker-compose.yml

@@ -12,6 +12,8 @@ services:
       - "5432"
     volumes:
       - dbdata:/var/lib/postgresql/data
+    restart: on-failure:10
+    oom_score_adj: -100
   webapp:
     image: opendronemap/webodm_webapp
     container_name: webapp
@@ -30,9 +32,12 @@ services:
       - WO_DEBUG
       - WO_BROKER
     restart: on-failure:10
+    oom_score_adj: 0
   broker:
     image: redis
     container_name: broker
+    restart: on-failure:10
+    oom_score_adj: -500
   worker:
     image: opendronemap/webodm_webapp
     container_name: worker
@@ -45,3 +50,5 @@ services:
     environment:
       - WO_BROKER
       - WO_DEBUG
+    restart: on-failure:10
+    oom_score_adj: 250

nodeodm/api_client.py

@@ -9,12 +9,11 @@ import os
 from urllib.parse import urlunparse
 from app.testwatch import TestWatch
 
-TIMEOUT = 30
-
 class ApiClient:
-    def __init__(self, host, port):
+    def __init__(self, host, port, timeout=30):
         self.host = host
         self.port = port
+        self.timeout = timeout
 
     def url(self, url):
         netloc = self.host if self.port == 80 else "{}:{}".format(self.host, self.port)
@@ -23,28 +22,28 @@ class ApiClient:
         return urlunparse(('http', netloc, url, '', '', ''))
 
     def info(self):
-        return requests.get(self.url('/info'), timeout=TIMEOUT).json()
+        return requests.get(self.url('/info'), timeout=self.timeout).json()
 
     def options(self):
-        return requests.get(self.url('/options'), timeout=TIMEOUT).json()
+        return requests.get(self.url('/options'), timeout=self.timeout).json()
 
     def task_info(self, uuid):
-        return requests.get(self.url('/task/{}/info').format(uuid), timeout=TIMEOUT).json()
+        return requests.get(self.url('/task/{}/info').format(uuid), timeout=self.timeout).json()
 
     @TestWatch.watch()
     def task_output(self, uuid, line = 0):
-        return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=TIMEOUT).json()
+        return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=self.timeout).json()
 
     def task_cancel(self, uuid):
-        return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=TIMEOUT).json()
+        return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=self.timeout).json()
 
     def task_remove(self, uuid):
-        return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=TIMEOUT).json()
+        return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=self.timeout).json()
 
     def task_restart(self, uuid, options = None):
         data = {'uuid': uuid}
         if options is not None: data['options'] = json.dumps(options)
-        return requests.post(self.url('/task/restart'), data=data, timeout=TIMEOUT).json()
+        return requests.post(self.url('/task/restart'), data=data, timeout=self.timeout).json()
 
     def task_download(self, uuid, asset):
         res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True)

nodeodm/models.py

@@ -66,7 +66,7 @@ class ProcessingNode(models.Model):
 
         :returns: True if information could be updated, False otherwise
         """
-        api_client = self.api_client()
+        api_client = self.api_client(timeout=5)
         try:
             info = api_client.info()
             self.api_version = info['version']
@@ -80,8 +80,8 @@ class ProcessingNode(models.Model):
         except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, json.decoder.JSONDecodeError, simplejson.JSONDecodeError):
             return False
 
-    def api_client(self):
+    def api_client(self, timeout=30):
-        return ApiClient(self.hostname, self.port)
+        return ApiClient(self.hostname, self.port, timeout)
 
     def get_available_options_json(self, pretty=False):
         """

webodm.sh

@@ -68,8 +68,8 @@ case $key in
     shift # past argument
     shift # past value
     ;;
-    --no-debug)
+    --debug)
-    export WO_DEBUG=NO
+    export WO_DEBUG=YES
     shift # past argument
     ;;
 	--broker)
@@ -108,7 +108,7 @@ usage(){
   echo "	--ssl-key	<path>	Manually specify a path to the private key file (.pem) to use with nginx to enable SSL (default: None)"
   echo "	--ssl-cert	<path>	Manually specify a path to the certificate file (.pem) to use with nginx to enable SSL (default: None)"
   echo "	--ssl-insecure-port-redirect	<port>	Insecure port number to redirect from when SSL is enabled (default: $DEFAULT_SSL_INSECURE_PORT_REDIRECT)"
-  echo "	--no-debug	Disable debug for production environments (default: disabled)"
+  echo "	--debug	Enable debug for development environments (default: disabled)"
   echo "	--broker	Set the URL used to connect to the celery broker (default: $DEFAULT_BROKER)"
   exit
 }

webodm/settings.py

@@ -326,6 +326,7 @@ CELERY_ACCEPT_CONTENT = ['json']
 CELERY_INCLUDE=['worker.tasks']
 CELERY_WORKER_REDIRECT_STDOUTS = False
 CELERY_WORKER_HIJACK_ROOT_LOGGER = False
+
 if TESTING:
     CELERY_TASK_ALWAYS_EAGER = True
 

worker.sh

@@ -52,7 +52,7 @@ start(){
 	action=$1
 
 	echo "Starting worker using broker at $WO_BROKER"
-	celery -A worker worker --loglevel=warn > /dev/null
+	celery -A worker worker --autoscale $(grep -c '^processor' /proc/cpuinfo),2 --max-tasks-per-child 1000 --loglevel=warn > /dev/null
 }
 
 start_scheduler(){

worker/tasks.py

@@ -79,7 +79,6 @@ def get_pending_tasks():
 @app.task
 def process_pending_tasks():
     tasks = get_pending_tasks()
-
     for task in tasks:
         process_task.delay(task.id)