Added support for server resize, in-browser resize, or no resize

app/api/tasks.py

@@ -145,7 +145,8 @@ class TaskViewSet(viewsets.ViewSet):
             raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
 
         with transaction.atomic():
-            task = models.Task.objects.create(project=project)
+            task = models.Task.objects.create(project=project,
+                                              pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
 
             for image in files:
                 models.ImageUpload.objects.create(task=task, image=image)
@@ -155,7 +156,9 @@ class TaskViewSet(viewsets.ViewSet):
             serializer.is_valid(raise_exception=True)
             serializer.save()
 
-            return Response(serializer.data, status=status.HTTP_201_CREATED)
+            worker_tasks.process_task.delay(task.id)
+
+        return Response(serializer.data, status=status.HTTP_201_CREATED)
 
 
     def update(self, request, pk=None, project_pk=None, partial=False):

app/migrations/0017_auto_20180219_1446.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2018-02-19 19:46
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0016_public_task_uuids'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='resize_to',
+            field=models.IntegerField(default=-1, help_text='When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.'),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='pending_action',
+            field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True),
+        ),
+    ]

app/models/task.py

@@ -4,6 +4,9 @@ import shutil
 import zipfile
 import uuid as uuid_module
 
+import piexif
+import re
+from PIL import Image
 from django.contrib.gis.gdal import GDALRaster
 from django.contrib.gis.gdal import OGRGeometry
 from django.contrib.gis.geos import GEOSGeometry
@@ -22,6 +25,10 @@ from nodeodm.models import ProcessingNode
 from webodm import settings
 from .project import Project
 
+from functools import partial
+from multiprocessing import cpu_count
+from concurrent.futures import ThreadPoolExecutor
+
 logger = logging.getLogger('app.logger')
 
 
@@ -57,6 +64,47 @@ def validate_task_options(value):
         raise ValidationError("Invalid options")
 
 
+
+def resize_image(image_path, resize_to):
+    try:
+        im = Image.open(image_path)
+        path, ext = os.path.splitext(image_path)
+        resized_image_path = os.path.join(path + '.resized' + ext)
+
+        width, height = im.size
+        max_side = max(width, height)
+        if max_side < resize_to:
+            logger.warning('You asked to make {} bigger ({} --> {}), but we are not going to do that.'.format(image_path, max_side, resize_to))
+            im.close()
+            return {'path': image_path, 'resize_ratio': 1}
+
+        ratio = float(resize_to) / float(max_side)
+        resized_width = int(width * ratio)
+        resized_height = int(height * ratio)
+
+        im.thumbnail((resized_width, resized_height), Image.LANCZOS)
+
+        if 'exif' in im.info:
+            exif_dict = piexif.load(im.info['exif'])
+            exif_dict['Exif'][piexif.ExifIFD.PixelXDimension] = resized_width
+            exif_dict['Exif'][piexif.ExifIFD.PixelYDimension] = resized_height
+            im.save(resized_image_path, "JPEG", exif=piexif.dump(exif_dict), quality=100)
+        else:
+            im.save(resized_image_path, "JPEG", quality=100)
+
+        im.close()
+
+        # Delete original image, rename resized image to original
+        os.remove(image_path)
+        os.rename(resized_image_path, image_path)
+
+        logger.info("Resized {} to {}x{}".format(image_path, resized_width, resized_height))
+    except IOError as e:
+        logger.warning("Cannot resize {}: {}.".format(image_path, str(e)))
+        return None
+
+    return {'path': image_path, 'resize_ratio': ratio}
+
 class Task(models.Model):
     ASSETS_MAP = {
             'all.zip': 'all.zip',
@@ -85,6 +133,7 @@ class Task(models.Model):
         (pending_actions.CANCEL, 'CANCEL'),
         (pending_actions.REMOVE, 'REMOVE'),
         (pending_actions.RESTART, 'RESTART'),
+        (pending_actions.RESIZE, 'RESIZE'),
     )
 
     id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False)
@@ -112,6 +161,7 @@ class Task(models.Model):
     pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.")
 
     public = models.BooleanField(default=False, help_text="A flag indicating whether this task is available to the public")
+    resize_to = models.IntegerField(default=-1, help_text="When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.")
 
 
     def __init__(self, *args, **kwargs):
@@ -227,6 +277,11 @@ class Task(models.Model):
         """
 
         try:
+            if self.pending_action == pending_actions.RESIZE:
+                self.resize_images()
+                self.pending_action = None
+                self.save()
+
             if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
                 # No processing node assigned and need to auto assign
                 if self.processing_node is None:
@@ -515,8 +570,31 @@ class Task(models.Model):
         self.pending_action = None
         self.save()
 
+
+    def resize_images(self):
+        """
+        Destructively resize this assets JPG images while retaining EXIF tags.
+        Resulting images are always converted to JPG.
+        TODO: add support for tiff files
+        :return list containing paths of resized images and resize ratios
+        """
+        if self.resize_to < 0:
+            logger.warning("We were asked to resize images to {}, this might be an error.".format(self.resize_to))
+            return []
+
+        directory = full_task_directory_path(self.id, self.project.id)
+        images_path = [os.path.join(directory, f) for f in os.listdir(directory) if
+                       re.match(r'.*\.jpe?g$', f, re.IGNORECASE)]
+
+        with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
+            resized_images = list(filter(lambda i: i is not None, executor.map(
+                partial(resize_image, resize_to=self.resize_to),
+                images_path)))
+
+        return resized_images
+
+
     class Meta:
         permissions = (
             ('view_task', 'Can view task'),
         )
-

app/pending_actions.py

@@ -1,3 +1,4 @@
 CANCEL = 1
 REMOVE = 2
-RESTART = 3
+RESTART = 3
+RESIZE = 4

app/static/app/js/classes/PendingActions.js

@@ -1,6 +1,7 @@
 const CANCEL = 1,
       REMOVE = 2,
-      RESTART = 3;
+      RESTART = 3,
+      RESIZE = 4;
 
 let pendingActions = {
     [CANCEL]: {
@@ -11,6 +12,9 @@ let pendingActions = {
     },
     [RESTART]: {
         descr: "Restarting..."
+    },
+    [RESIZE]: {
+      descr: "Resizing images..."
     }
 };
 
@@ -18,6 +22,7 @@ export default {
     CANCEL: CANCEL,
     REMOVE: REMOVE,
     RESTART: RESTART,
+    RESIZE: RESIZE,
 
     description: function(pendingAction) {
       if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr;

app/static/app/js/classes/ResizeModes.js

@@ -0,0 +1,26 @@
+const dict = [
+  {k: 'NO', v: 0, human: "No"}, // Don't resize
+  {k: 'YES', v: 1, human: "Yes"}, // Resize on server
+  {k: 'YESINBROWSER', v: 2, human: "Yes (In browser)"} // Resize on browser
+];
+
+const exp = {
+  all: () => dict.map(d => d.v),
+  fromString: (s) => {
+    let v = parseInt(s);
+    if (!isNaN(v) && v >= 0 && v <= 2) return v;
+    else return 0;
+  },
+  toHuman: (v) => {
+    for (let i in dict){
+      if (dict[i].v === v) return dict[i].human;
+    }
+    throw new Error("Invalid value: " + v);
+  }
+};
+dict.forEach(en => {
+  exp[en.k] = en.v;
+});
+
+export default exp;
+

app/static/app/js/components/NewTaskPanel.jsx

@@ -3,6 +3,7 @@ import React from 'react';
 import EditTaskForm from './EditTaskForm';
 import PropTypes from 'prop-types';
 import Storage from '../classes/Storage';
+import ResizeModes from '../classes/ResizeModes';
 
 class NewTaskPanel extends React.Component {
   static defaultProps = {
@@ -25,14 +26,14 @@ class NewTaskPanel extends React.Component {
     this.state = {
       name: props.name,
       editTaskFormLoaded: false,
-      resize: Storage.getItem('do_resize') !== null ? Storage.getItem('do_resize') == "1" : true,
+      resizeMode: Storage.getItem('resize_mode') === null ? ResizeModes.YES : ResizeModes.fromString(Storage.getItem('resize_mode')),
       resizeSize: parseInt(Storage.getItem('resize_size')) || 2048
     };
 
     this.save = this.save.bind(this);
     this.handleFormTaskLoaded = this.handleFormTaskLoaded.bind(this);
     this.getTaskInfo = this.getTaskInfo.bind(this);
-    this.setResize = this.setResize.bind(this);
+    this.setResizeMode = this.setResizeMode.bind(this);
     this.handleResizeSizeChange = this.handleResizeSizeChange.bind(this);
   }
 
@@ -40,7 +41,7 @@ class NewTaskPanel extends React.Component {
     e.preventDefault();
     this.taskForm.saveLastPresetToStorage();
     Storage.setItem('resize_size', this.state.resizeSize);
-    Storage.setItem('do_resize', this.state.resize ? "1" : "0");
+    Storage.setItem('resize_mode', this.state.resizeMode);
     if (this.props.onSave) this.props.onSave(this.getTaskInfo());
   }
 
@@ -54,13 +55,14 @@ class NewTaskPanel extends React.Component {
 
   getTaskInfo(){
     return Object.assign(this.taskForm.getTaskInfo(), {
-      resizeTo: (this.state.resize && this.state.resizeSize > 0) ? this.state.resizeSize : null
+      resizeSize: this.state.resizeSize,
+      resizeMode: this.state.resizeMode 
     });
   }
 
-  setResize(flag){
+  setResizeMode(v){
     return e => {
-      this.setState({resize: flag});
+      this.setState({resizeMode: v});
     }
   }
 
@@ -91,23 +93,19 @@ class NewTaskPanel extends React.Component {
               <div className="col-sm-10">
                 <div className="btn-group">
                   <button type="button" className="btn btn-default dropdown-toggle" data-toggle="dropdown">
-                    {this.state.resize ? 
+                    {ResizeModes.toHuman(this.state.resizeMode)} <span className="caret"></span>
-                     "Yes" : "No"} <span className="caret"></span>
                   </button>
                   <ul className="dropdown-menu">
-                    <li>
+                    {ResizeModes.all().map(mode =>
-                      <a href="javascript:void(0);" 
+                      <li key={mode}>
-                          onClick={this.setResize(true)}>
+                        <a href="javascript:void(0);" 
-                          <i style={{opacity: this.state.resize ? 1 : 0}} className="fa fa-check"></i> Yes</a>
+                            onClick={this.setResizeMode(mode)}>
-                    </li>
+                            <i style={{opacity: this.state.resizeMode === mode ? 1 : 0}} className="fa fa-check"></i> {ResizeModes.toHuman(mode)}</a>
-                    <li>
+                      </li>
-                      <a href="javascript:void(0);" 
+                    )}
-                          onClick={this.setResize(false)}>
-                          <i style={{opacity: !this.state.resize ? 1 : 0}} className="fa fa-check"></i> Skip</a>
-                    </li>
                   </ul>
                 </div>
-                <div className={"resize-control " + (!this.state.resize ? "hide" : "")}>
+                <div className={"resize-control " + (this.state.resizeMode === ResizeModes.NO ? "hide" : "")}>
                   <input 
                     type="number" 
                     step="100"

app/static/app/js/components/ProjectListItem.jsx

@@ -11,6 +11,7 @@ import Dropzone from '../vendor/dropzone';
 import csrf from '../django/csrf';
 import HistoryNav from '../classes/HistoryNav';
 import PropTypes from 'prop-types';
+import ResizeModes from '../classes/ResizeModes';
 import $ from 'jquery';
 
 class ProjectListItem extends React.Component {
@@ -180,6 +181,10 @@ class ProjectListItem extends React.Component {
           if (!formData.has || !formData.has("options")) formData.append("options", JSON.stringify(taskInfo.options));
           if (!formData.has || !formData.has("processing_node")) formData.append("processing_node", taskInfo.selectedNode.id);
           if (!formData.has || !formData.has("auto_processing_node")) formData.append("auto_processing_node", taskInfo.selectedNode.key == "auto");
+
+          if (taskInfo.resizeMode === ResizeModes.YES){
+            if (!formData.has || !formData.has("resize_to")) formData.append("resize_to", taskInfo.resizeSize);
+          }
         });
     }
   }
@@ -225,8 +230,8 @@ class ProjectListItem extends React.Component {
     this.dz._taskInfo = taskInfo; // Allow us to access the task info from dz
 
     // Update dropzone settings
-    if (taskInfo.resizeTo !== null){
+    if (taskInfo.resizeMode === ResizeModes.YESINBROWSER){
-      this.dz.options.resizeWidth = taskInfo.resizeTo;
+      this.dz.options.resizeWidth = taskInfo.resizeSize;
       this.dz.options.resizeQuality = 1.0;
 
       this.setUploadState({resizing: true, editing: false});

app/static/app/js/components/TaskListItem.jsx

@@ -351,7 +351,7 @@ class TaskListItem extends React.Component {
     let status = statusCodes.description(task.status);
     if (status === "") status = "Uploading images";
 
-    if (!task.processing_node) status = "";
+    if (!task.processing_node) status = "Waiting for a node...";
     if (task.pending_action !== null) status = pendingActions.description(task.pending_action);
 
     let expanded = "";

start.sh

@@ -69,6 +69,8 @@ if [ "$WO_SSL" = "YES" ]; then
     proto="https"
 fi
 
+./worker.sh scheduler start
+
 congrats(){
     (sleep 5; echo
     echo -e "\033[92m"      

webodm/settings.py

@@ -322,6 +322,8 @@ CELERY_TASK_SERIALIZER = 'json'
 CELERY_RESULT_SERIALIZER = 'json'
 CELERY_ACCEPT_CONTENT = ['json']
 CELERY_INCLUDE=['worker.tasks']
+CELERY_WORKER_REDIRECT_STDOUTS = False
+CELERY_WORKER_HIJACK_ROOT_LOGGER = False
 if TESTING:
     CELERY_TASK_ALWAYS_EAGER = True
 

worker.sh

@@ -9,7 +9,9 @@ usage(){
   echo "This program manages the background worker processes. WebODM requires at least one background process worker to be running at all times."
   echo 
   echo "Command list:"
-  echo "	start		Start background worker"
+  echo "	start				Start background worker"
+  echo "	scheduler start		Start background worker scheduler"
+  echo "	scheduler stop 		Stop background worker scheduler"
   exit
 }
 
@@ -51,12 +53,41 @@ start(){
 	action=$1
 
 	echo "Starting worker using broker at $WO_BROKER"
-	celery -A worker worker --loglevel=info
+	celery -A worker worker --loglevel=warn > /dev/null
+}
+
+start_scheduler(){
+	stop_scheduler
+	if [[ ! -f ./celerybeat.pid ]]; then
+		celery -A worker beat &
+	else
+		echo "Scheduler already running (celerybeat.pid exists)."
+	fi
+}
+
+stop_scheduler(){
+	if [[ -f ./celerybeat.pid ]]; then
+		kill -9 $(cat ./celerybeat.pid) 2>/dev/null
+		rm ./celerybeat.pid 2>/dev/null
+		echo "Scheduler has shutdown."
+	else
+		echo "Scheduler is not running."
+	fi
 }
 
 if [[ $1 = "start" ]]; then
 	environment_check
 	start
+elif [[ $1 = "scheduler" ]]; then
+	if [[ $2 = "start" ]]; then
+		environment_check
+		start_scheduler
+	elif [[ $2 = "stop" ]]; then
+		environment_check
+		stop_scheduler
+	else
+		usage
+	fi
 else
 	usage
 fi

worker/tasks.py

@@ -1,24 +1,17 @@
-import os
 import traceback
 
-import re
+from celery.utils.log import get_task_logger
-
-import piexif
 from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
 from django.db.models import Count
 from django.db.models import Q
 
 from app.models import Project
 from app.models import Task
-from webodm import settings
 from nodeodm import status_codes
 from nodeodm.models import ProcessingNode
+from webodm import settings
 from .celery import app
-from celery.utils.log import get_task_logger
-from django.db import transaction
-from PIL import Image
-from functools import partial
-from multiprocessing import Pool, cpu_count
 
 logger = get_task_logger(__name__)
 
@@ -84,57 +77,3 @@ def process_pending_tasks():
 
     for task in tasks:
         process_task.delay(task.id)
-
-
-@app.task
-def resize_image(image_path, resize_to):
-    try:
-        exif_dict = piexif.load(image_path)
-        im = Image.open(image_path)
-        path, ext = os.path.splitext(image_path)
-        resized_image_path = os.path.join(path + '.resized' + ext)
-
-        width, height = im.size
-        max_side = max(width, height)
-        if max_side < resize_to:
-            logger.warning('We are making {} bigger ({} --> {})'.format(image_path, max_side, resize_to))
-
-        ratio = float(resize_to) / float(max_side)
-        resized_width = int(width * ratio)
-        resized_height = int(height * ratio)
-
-        im.thumbnail((resized_width, resized_height), Image.LANCZOS)
-
-        if len(exif_dict['Exif']) > 0:
-            exif_dict['Exif'][piexif.ExifIFD.PixelXDimension] = resized_width
-            exif_dict['Exif'][piexif.ExifIFD.PixelYDimension] = resized_height
-            im.save(resized_image_path, "JPEG", exif=piexif.dump(exif_dict), quality=100)
-        else:
-            im.save(resized_image_path, "JPEG", quality=100)
-
-        im.close()
-
-        # Delete original image, rename resized image to original
-        os.remove(image_path)
-        os.rename(resized_image_path, image_path)
-
-        logger.info("Resized {}".format(os.path.basename(resized_image_path)))
-    except IOError as e:
-        logger.warning("Cannot resize {}: {}.".format(image_path, str(e)))
-        return None
-
-    return image_path
-
-@app.task
-def resize_images(directory, resize_to):
-    """
-    Destructively resize a directory of JPG images while retaining EXIF tags.
-    Resulting images are always converted to JPG.
-    TODO: add support for tiff files
-    :return list containing paths of resized images
-    """
-    images_path = [os.path.join(directory, f) for f in os.listdir(directory) if re.match(r'.*\.jpe?g$', f, re.IGNORECASE)]
-    resized_images = list(filter(lambda i: i is not None, Pool(cpu_count()).map(
-        partial(resize_image, resize_to=resize_to),
-        images_path)))
-    return resized_images