Merge pull request #1669 from treescape-io/improved_worker_count

Configurable worker count. Closes #1668.

docker-compose.yml

@@ -34,6 +34,7 @@ services:
       - WO_DEV
       - WO_DEV_WATCH_PLUGINS
       - WO_SECRET_KEY
+      - WEB_CONCURRENCY
     restart: unless-stopped
     oom_score_adj: 0
   broker:
@@ -54,5 +55,6 @@ services:
       - WO_BROKER
       - WO_DEBUG
       - WO_SECRET_KEY
+      - WEB_CONCURRENCY
     restart: unless-stopped
     oom_score_adj: 250

requirements.txt

@@ -20,7 +20,7 @@ djangorestframework-guardian==0.3.0
 drf-nested-routers==0.11.1
 funcsigs==1.0.2
 futures==3.1.1
-gunicorn==19.8.0
+gunicorn==23.0.0
 geodeep==0.9.8
 itypes==1.1.0
 kombu==4.6.7

start.sh

@@ -144,10 +144,22 @@ else
         conf="nginx-ssl.conf"
     fi
 
+    # Only set if `WEB_CONCURRENCY` is not defined, allows overriding.
+    # See: https://docs.gunicorn.org/en/latest/settings.html#workers
+    if [ -z "$WEB_CONCURRENCY" ]; then
+        # Default gunicorn worker count to 2 x availabe cores + 1
+        # nproc gives _available_ (rather than physicall present) CPU cores.
+        CPU_COUNT=$(nproc)
+        export WEB_CONCURRENCY=$((2*$CPU_COUNT+1))
+        echo "Setting WEB_CONCURRENCY based on $CPU_COUNT available CPU cores to start $WEB_CONCURRENCY gunicorn workers."
+    else
+        echo "Using pre-defined WEB_CONCURRENCY override to start $WEB_CONCURRENCY gunicorn workers."
+    fi
+
     congrats
 
     nginx -c $(pwd)/nginx/$conf
-    gunicorn webodm.wsgi --bind unix:/tmp/gunicorn.sock --timeout 300000 --max-requests 500 --workers $((2*$(grep -c '^processor' /proc/cpuinfo)+1)) --preload
+    gunicorn webodm.wsgi --bind unix:/tmp/gunicorn.sock --timeout 300000 --max-requests 500 --workers $WEB_CONCURRENCY --preload
 fi
 
 # If this is executed, it means the previous command failed, don't display the congratulations message

worker.sh

@@ -45,6 +45,18 @@ environment_check(){
 		echo -e "\033[91mWO_BROKER environment variable is not set. Defaulting to redis://localhost\033[39m"
 		export WO_BROKER=redis://localhost
 	fi
+
+  # Only set if `WEB_CONCURRENCY` is not defined, allows overriding.
+  # See: https://docs.gunicorn.org/en/latest/settings.html#workers
+  if [ -z "$WEB_CONCURRENCY" ]; then
+      # Default gunicorn worker count to 2 x availabe cores + 1
+      # nproc gives _available_ (rather than physicall present) CPU cores.
+      CPU_COUNT=$(nproc)
+      export WEB_CONCURRENCY=$((2*$CPU_COUNT+1))
+      echo "Setting WEB_CONCURRENCY based on $CPU_COUNT available CPU cores to start $WEB_CONCURRENCY gunicorn workers."
+  else
+      echo "Using pre-defined WEB_CONCURRENCY override to start $WEB_CONCURRENCY gunicorn workers."
+  fi
 }
 
 
@@ -52,7 +64,7 @@ start(){
 	action=$1
 
 	echo "Starting worker using broker at $WO_BROKER"
-	celery -A worker worker --autoscale $(grep -c '^processor' /proc/cpuinfo),2 --max-tasks-per-child 1000 --loglevel=warn > /dev/null
+	celery -A worker worker --autoscale $WEB_CONCURRENCY,2 --max-tasks-per-child 1000 --loglevel=warn > /dev/null
 }
 
 start_scheduler(){