From 273594240992363a13e8100f32a2c156d4a228f9 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 19 Feb 2018 15:50:26 -0500 Subject: [PATCH] Added support for server resize, in-browser resize, or no resize --- app/api/tasks.py | 7 +- app/migrations/0017_auto_20180219_1446.py | 25 ++++++ app/models/task.py | 80 ++++++++++++++++++- app/pending_actions.py | 3 +- app/static/app/js/classes/PendingActions.js | 7 +- app/static/app/js/classes/ResizeModes.js | 26 ++++++ app/static/app/js/components/NewTaskPanel.jsx | 36 ++++----- .../app/js/components/ProjectListItem.jsx | 9 ++- app/static/app/js/components/TaskListItem.jsx | 2 +- start.sh | 2 + webodm/settings.py | 2 + worker.sh | 35 +++++++- worker/tasks.py | 67 +--------------- 13 files changed, 208 insertions(+), 93 deletions(-) create mode 100644 app/migrations/0017_auto_20180219_1446.py create mode 100644 app/static/app/js/classes/ResizeModes.js diff --git a/app/api/tasks.py b/app/api/tasks.py index a0bd0993..994e502b 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -145,7 +145,8 @@ class TaskViewSet(viewsets.ViewSet): raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images") with transaction.atomic(): - task = models.Task.objects.create(project=project) + task = models.Task.objects.create(project=project, + pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None) for image in files: models.ImageUpload.objects.create(task=task, image=image) @@ -155,7 +156,9 @@ class TaskViewSet(viewsets.ViewSet): serializer.is_valid(raise_exception=True) serializer.save() - return Response(serializer.data, status=status.HTTP_201_CREATED) + worker_tasks.process_task.delay(task.id) + + return Response(serializer.data, status=status.HTTP_201_CREATED) def update(self, request, pk=None, project_pk=None, partial=False): diff --git a/app/migrations/0017_auto_20180219_1446.py b/app/migrations/0017_auto_20180219_1446.py new file mode 100644 index 00000000..c97f8e6c --- /dev/null +++ b/app/migrations/0017_auto_20180219_1446.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2018-02-19 19:46 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0016_public_task_uuids'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='resize_to', + field=models.IntegerField(default=-1, help_text='When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.'), + ), + migrations.AlterField( + model_name='task', + name='pending_action', + field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True), + ), + ] diff --git a/app/models/task.py b/app/models/task.py index 4808b561..812b4eb2 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -4,6 +4,9 @@ import shutil import zipfile import uuid as uuid_module +import piexif +import re +from PIL import Image from django.contrib.gis.gdal import GDALRaster from django.contrib.gis.gdal import OGRGeometry from django.contrib.gis.geos import GEOSGeometry @@ -22,6 +25,10 @@ from nodeodm.models import ProcessingNode from webodm import settings from .project import Project +from functools import partial +from multiprocessing import cpu_count +from concurrent.futures import ThreadPoolExecutor + logger = logging.getLogger('app.logger') @@ -57,6 +64,47 @@ def validate_task_options(value): raise ValidationError("Invalid options") + +def resize_image(image_path, resize_to): + try: + im = Image.open(image_path) + path, ext = os.path.splitext(image_path) + resized_image_path = os.path.join(path + '.resized' + ext) + + width, height = im.size + max_side = max(width, height) + if max_side < resize_to: + logger.warning('You asked to make {} bigger ({} --> {}), but we are not going to do that.'.format(image_path, max_side, resize_to)) + im.close() + return {'path': image_path, 'resize_ratio': 1} + + ratio = float(resize_to) / float(max_side) + resized_width = int(width * ratio) + resized_height = int(height * ratio) + + im.thumbnail((resized_width, resized_height), Image.LANCZOS) + + if 'exif' in im.info: + exif_dict = piexif.load(im.info['exif']) + exif_dict['Exif'][piexif.ExifIFD.PixelXDimension] = resized_width + exif_dict['Exif'][piexif.ExifIFD.PixelYDimension] = resized_height + im.save(resized_image_path, "JPEG", exif=piexif.dump(exif_dict), quality=100) + else: + im.save(resized_image_path, "JPEG", quality=100) + + im.close() + + # Delete original image, rename resized image to original + os.remove(image_path) + os.rename(resized_image_path, image_path) + + logger.info("Resized {} to {}x{}".format(image_path, resized_width, resized_height)) + except IOError as e: + logger.warning("Cannot resize {}: {}.".format(image_path, str(e))) + return None + + return {'path': image_path, 'resize_ratio': ratio} + class Task(models.Model): ASSETS_MAP = { 'all.zip': 'all.zip', @@ -85,6 +133,7 @@ class Task(models.Model): (pending_actions.CANCEL, 'CANCEL'), (pending_actions.REMOVE, 'REMOVE'), (pending_actions.RESTART, 'RESTART'), + (pending_actions.RESIZE, 'RESIZE'), ) id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False) @@ -112,6 +161,7 @@ class Task(models.Model): pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.") public = models.BooleanField(default=False, help_text="A flag indicating whether this task is available to the public") + resize_to = models.IntegerField(default=-1, help_text="When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.") def __init__(self, *args, **kwargs): @@ -227,6 +277,11 @@ class Task(models.Model): """ try: + if self.pending_action == pending_actions.RESIZE: + self.resize_images() + self.pending_action = None + self.save() + if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]: # No processing node assigned and need to auto assign if self.processing_node is None: @@ -515,8 +570,31 @@ class Task(models.Model): self.pending_action = None self.save() + + def resize_images(self): + """ + Destructively resize this assets JPG images while retaining EXIF tags. + Resulting images are always converted to JPG. + TODO: add support for tiff files + :return list containing paths of resized images and resize ratios + """ + if self.resize_to < 0: + logger.warning("We were asked to resize images to {}, this might be an error.".format(self.resize_to)) + return [] + + directory = full_task_directory_path(self.id, self.project.id) + images_path = [os.path.join(directory, f) for f in os.listdir(directory) if + re.match(r'.*\.jpe?g$', f, re.IGNORECASE)] + + with ThreadPoolExecutor(max_workers=cpu_count()) as executor: + resized_images = list(filter(lambda i: i is not None, executor.map( + partial(resize_image, resize_to=self.resize_to), + images_path))) + + return resized_images + + class Meta: permissions = ( ('view_task', 'Can view task'), ) - diff --git a/app/pending_actions.py b/app/pending_actions.py index b5c7bdd8..79c0cc6b 100644 --- a/app/pending_actions.py +++ b/app/pending_actions.py @@ -1,3 +1,4 @@ CANCEL = 1 REMOVE = 2 -RESTART = 3 \ No newline at end of file +RESTART = 3 +RESIZE = 4 diff --git a/app/static/app/js/classes/PendingActions.js b/app/static/app/js/classes/PendingActions.js index 4e67b57c..b69f1728 100644 --- a/app/static/app/js/classes/PendingActions.js +++ b/app/static/app/js/classes/PendingActions.js @@ -1,6 +1,7 @@ const CANCEL = 1, REMOVE = 2, - RESTART = 3; + RESTART = 3, + RESIZE = 4; let pendingActions = { [CANCEL]: { @@ -11,6 +12,9 @@ let pendingActions = { }, [RESTART]: { descr: "Restarting..." + }, + [RESIZE]: { + descr: "Resizing images..." } }; @@ -18,6 +22,7 @@ export default { CANCEL: CANCEL, REMOVE: REMOVE, RESTART: RESTART, + RESIZE: RESIZE, description: function(pendingAction) { if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr; diff --git a/app/static/app/js/classes/ResizeModes.js b/app/static/app/js/classes/ResizeModes.js new file mode 100644 index 00000000..2ceb52e5 --- /dev/null +++ b/app/static/app/js/classes/ResizeModes.js @@ -0,0 +1,26 @@ +const dict = [ + {k: 'NO', v: 0, human: "No"}, // Don't resize + {k: 'YES', v: 1, human: "Yes"}, // Resize on server + {k: 'YESINBROWSER', v: 2, human: "Yes (In browser)"} // Resize on browser +]; + +const exp = { + all: () => dict.map(d => d.v), + fromString: (s) => { + let v = parseInt(s); + if (!isNaN(v) && v >= 0 && v <= 2) return v; + else return 0; + }, + toHuman: (v) => { + for (let i in dict){ + if (dict[i].v === v) return dict[i].human; + } + throw new Error("Invalid value: " + v); + } +}; +dict.forEach(en => { + exp[en.k] = en.v; +}); + +export default exp; + diff --git a/app/static/app/js/components/NewTaskPanel.jsx b/app/static/app/js/components/NewTaskPanel.jsx index e086b566..ecd4189f 100644 --- a/app/static/app/js/components/NewTaskPanel.jsx +++ b/app/static/app/js/components/NewTaskPanel.jsx @@ -3,6 +3,7 @@ import React from 'react'; import EditTaskForm from './EditTaskForm'; import PropTypes from 'prop-types'; import Storage from '../classes/Storage'; +import ResizeModes from '../classes/ResizeModes'; class NewTaskPanel extends React.Component { static defaultProps = { @@ -25,14 +26,14 @@ class NewTaskPanel extends React.Component { this.state = { name: props.name, editTaskFormLoaded: false, - resize: Storage.getItem('do_resize') !== null ? Storage.getItem('do_resize') == "1" : true, + resizeMode: Storage.getItem('resize_mode') === null ? ResizeModes.YES : ResizeModes.fromString(Storage.getItem('resize_mode')), resizeSize: parseInt(Storage.getItem('resize_size')) || 2048 }; this.save = this.save.bind(this); this.handleFormTaskLoaded = this.handleFormTaskLoaded.bind(this); this.getTaskInfo = this.getTaskInfo.bind(this); - this.setResize = this.setResize.bind(this); + this.setResizeMode = this.setResizeMode.bind(this); this.handleResizeSizeChange = this.handleResizeSizeChange.bind(this); } @@ -40,7 +41,7 @@ class NewTaskPanel extends React.Component { e.preventDefault(); this.taskForm.saveLastPresetToStorage(); Storage.setItem('resize_size', this.state.resizeSize); - Storage.setItem('do_resize', this.state.resize ? "1" : "0"); + Storage.setItem('resize_mode', this.state.resizeMode); if (this.props.onSave) this.props.onSave(this.getTaskInfo()); } @@ -54,13 +55,14 @@ class NewTaskPanel extends React.Component { getTaskInfo(){ return Object.assign(this.taskForm.getTaskInfo(), { - resizeTo: (this.state.resize && this.state.resizeSize > 0) ? this.state.resizeSize : null + resizeSize: this.state.resizeSize, + resizeMode: this.state.resizeMode }); } - setResize(flag){ + setResizeMode(v){ return e => { - this.setState({resize: flag}); + this.setState({resizeMode: v}); } } @@ -91,23 +93,19 @@ class NewTaskPanel extends React.Component {
-
+
/dev/null +} + +start_scheduler(){ + stop_scheduler + if [[ ! -f ./celerybeat.pid ]]; then + celery -A worker beat & + else + echo "Scheduler already running (celerybeat.pid exists)." + fi +} + +stop_scheduler(){ + if [[ -f ./celerybeat.pid ]]; then + kill -9 $(cat ./celerybeat.pid) 2>/dev/null + rm ./celerybeat.pid 2>/dev/null + echo "Scheduler has shutdown." + else + echo "Scheduler is not running." + fi } if [[ $1 = "start" ]]; then environment_check start +elif [[ $1 = "scheduler" ]]; then + if [[ $2 = "start" ]]; then + environment_check + start_scheduler + elif [[ $2 = "stop" ]]; then + environment_check + stop_scheduler + else + usage + fi else usage fi diff --git a/worker/tasks.py b/worker/tasks.py index a57aa7e3..672993e7 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -1,24 +1,17 @@ -import os import traceback -import re - -import piexif +from celery.utils.log import get_task_logger from django.core.exceptions import ObjectDoesNotExist +from django.db import transaction from django.db.models import Count from django.db.models import Q from app.models import Project from app.models import Task -from webodm import settings from nodeodm import status_codes from nodeodm.models import ProcessingNode +from webodm import settings from .celery import app -from celery.utils.log import get_task_logger -from django.db import transaction -from PIL import Image -from functools import partial -from multiprocessing import Pool, cpu_count logger = get_task_logger(__name__) @@ -84,57 +77,3 @@ def process_pending_tasks(): for task in tasks: process_task.delay(task.id) - - -@app.task -def resize_image(image_path, resize_to): - try: - exif_dict = piexif.load(image_path) - im = Image.open(image_path) - path, ext = os.path.splitext(image_path) - resized_image_path = os.path.join(path + '.resized' + ext) - - width, height = im.size - max_side = max(width, height) - if max_side < resize_to: - logger.warning('We are making {} bigger ({} --> {})'.format(image_path, max_side, resize_to)) - - ratio = float(resize_to) / float(max_side) - resized_width = int(width * ratio) - resized_height = int(height * ratio) - - im.thumbnail((resized_width, resized_height), Image.LANCZOS) - - if len(exif_dict['Exif']) > 0: - exif_dict['Exif'][piexif.ExifIFD.PixelXDimension] = resized_width - exif_dict['Exif'][piexif.ExifIFD.PixelYDimension] = resized_height - im.save(resized_image_path, "JPEG", exif=piexif.dump(exif_dict), quality=100) - else: - im.save(resized_image_path, "JPEG", quality=100) - - im.close() - - # Delete original image, rename resized image to original - os.remove(image_path) - os.rename(resized_image_path, image_path) - - logger.info("Resized {}".format(os.path.basename(resized_image_path))) - except IOError as e: - logger.warning("Cannot resize {}: {}.".format(image_path, str(e))) - return None - - return image_path - -@app.task -def resize_images(directory, resize_to): - """ - Destructively resize a directory of JPG images while retaining EXIF tags. - Resulting images are always converted to JPG. - TODO: add support for tiff files - :return list containing paths of resized images - """ - images_path = [os.path.join(directory, f) for f in os.listdir(directory) if re.match(r'.*\.jpe?g$', f, re.IGNORECASE)] - resized_images = list(filter(lambda i: i is not None, Pool(cpu_count()).map( - partial(resize_image, resize_to=resize_to), - images_path))) - return resized_images \ No newline at end of file