mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Added support for server resize, in-browser resize, or no resize

pull/384/head
Piero Toffanin 2018-02-19 15:50:26 -05:00
rodzic 3415220565
commit 2735942409
13 zmienionych plików z 208 dodań i 93 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

7
app/api/tasks.py
Wyświetl plik

@ -145,7 +145,8 @@ class TaskViewSet(viewsets.ViewSet):
raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
with transaction.atomic():
task = models.Task.objects.create(project=project)
task = models.Task.objects.create(project=project,
pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
for image in files:
models.ImageUpload.objects.create(task=task, image=image)
@ -155,7 +156,9 @@ class TaskViewSet(viewsets.ViewSet):
serializer.is_valid(raise_exception=True)
serializer.save()
return Response(serializer.data, status=status.HTTP_201_CREATED)
worker_tasks.process_task.delay(task.id)
return Response(serializer.data, status=status.HTTP_201_CREATED)
def update(self, request, pk=None, project_pk=None, partial=False):

25
app/migrations/0017_auto_20180219_1446.py 100644
Wyświetl plik

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.7 on 2018-02-19 19:46
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app', '0016_public_task_uuids'),
]
operations = [
migrations.AddField(
model_name='task',
name='resize_to',
field=models.IntegerField(default=-1, help_text='When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.'),
),
migrations.AlterField(
model_name='task',
name='pending_action',
field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True),
),
]

80
app/models/task.py
Wyświetl plik

@ -4,6 +4,9 @@ import shutil
import zipfile
import uuid as uuid_module
import piexif
import re
from PIL import Image
from django.contrib.gis.gdal import GDALRaster
from django.contrib.gis.gdal import OGRGeometry
from django.contrib.gis.geos import GEOSGeometry
@ -22,6 +25,10 @@ from nodeodm.models import ProcessingNode
from webodm import settings
from .project import Project
from functools import partial
from multiprocessing import cpu_count
from concurrent.futures import ThreadPoolExecutor
logger = logging.getLogger('app.logger')
@ -57,6 +64,47 @@ def validate_task_options(value):
raise ValidationError("Invalid options")
def resize_image(image_path, resize_to):
try:
im = Image.open(image_path)
path, ext = os.path.splitext(image_path)
resized_image_path = os.path.join(path + '.resized' + ext)
width, height = im.size
max_side = max(width, height)
if max_side < resize_to:
logger.warning('You asked to make {} bigger ({} --> {}), but we are not going to do that.'.format(image_path, max_side, resize_to))
im.close()
return {'path': image_path, 'resize_ratio': 1}
ratio = float(resize_to) / float(max_side)
resized_width = int(width * ratio)
resized_height = int(height * ratio)
im.thumbnail((resized_width, resized_height), Image.LANCZOS)
if 'exif' in im.info:
exif_dict = piexif.load(im.info['exif'])
exif_dict['Exif'][piexif.ExifIFD.PixelXDimension] = resized_width
exif_dict['Exif'][piexif.ExifIFD.PixelYDimension] = resized_height
im.save(resized_image_path, "JPEG", exif=piexif.dump(exif_dict), quality=100)
else:
im.save(resized_image_path, "JPEG", quality=100)
im.close()
# Delete original image, rename resized image to original
os.remove(image_path)
os.rename(resized_image_path, image_path)
logger.info("Resized {} to {}x{}".format(image_path, resized_width, resized_height))
except IOError as e:
logger.warning("Cannot resize {}: {}.".format(image_path, str(e)))
return None
return {'path': image_path, 'resize_ratio': ratio}
class Task(models.Model):
ASSETS_MAP = {
'all.zip': 'all.zip',
@ -85,6 +133,7 @@ class Task(models.Model):
(pending_actions.CANCEL, 'CANCEL'),
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
(pending_actions.RESIZE, 'RESIZE'),
)
id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False)
@ -112,6 +161,7 @@ class Task(models.Model):
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.")
public = models.BooleanField(default=False, help_text="A flag indicating whether this task is available to the public")
resize_to = models.IntegerField(default=-1, help_text="When set to a value different than -1, indicates that the images for this task have been / will be resized to the size specified here before processing.")
def __init__(self, *args, **kwargs):
@ -227,6 +277,11 @@ class Task(models.Model):
"""
try:
if self.pending_action == pending_actions.RESIZE:
self.resize_images()
self.pending_action = None
self.save()
if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
# No processing node assigned and need to auto assign
if self.processing_node is None:
@ -515,8 +570,31 @@ class Task(models.Model):
self.pending_action = None
self.save()
def resize_images(self):
"""
Destructively resize this assets JPG images while retaining EXIF tags.
Resulting images are always converted to JPG.
TODO: add support for tiff files
:return list containing paths of resized images and resize ratios
"""
if self.resize_to < 0:
logger.warning("We were asked to resize images to {}, this might be an error.".format(self.resize_to))
return []
directory = full_task_directory_path(self.id, self.project.id)
images_path = [os.path.join(directory, f) for f in os.listdir(directory) if
re.match(r'.*\.jpe?g$', f, re.IGNORECASE)]
with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
resized_images = list(filter(lambda i: i is not None, executor.map(
partial(resize_image, resize_to=self.resize_to),
images_path)))
return resized_images
class Meta:
permissions = (
('view_task', 'Can view task'),
)

3
app/pending_actions.py
Wyświetl plik

@ -1,3 +1,4 @@
CANCEL = 1
REMOVE = 2
RESTART = 3
RESTART = 3
RESIZE = 4

7
app/static/app/js/classes/PendingActions.js
Wyświetl plik

@ -1,6 +1,7 @@
const CANCEL = 1,
REMOVE = 2,
RESTART = 3;
RESTART = 3,
RESIZE = 4;
let pendingActions = {
[CANCEL]: {
@ -11,6 +12,9 @@ let pendingActions = {
},
[RESTART]: {
descr: "Restarting..."
},
[RESIZE]: {
descr: "Resizing images..."
}
};
@ -18,6 +22,7 @@ export default {
CANCEL: CANCEL,
REMOVE: REMOVE,
RESTART: RESTART,
RESIZE: RESIZE,
description: function(pendingAction) {
if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr;

26
app/static/app/js/classes/ResizeModes.js 100644
Wyświetl plik

@ -0,0 +1,26 @@
const dict = [
{k: 'NO', v: 0, human: "No"}, // Don't resize
{k: 'YES', v: 1, human: "Yes"}, // Resize on server
{k: 'YESINBROWSER', v: 2, human: "Yes (In browser)"} // Resize on browser
];
const exp = {
all: () => dict.map(d => d.v),
fromString: (s) => {
let v = parseInt(s);
if (!isNaN(v) && v >= 0 && v <= 2) return v;
else return 0;
},
toHuman: (v) => {
for (let i in dict){
if (dict[i].v === v) return dict[i].human;
}
throw new Error("Invalid value: " + v);
}
};
dict.forEach(en => {
exp[en.k] = en.v;
});
export default exp;

36
app/static/app/js/components/NewTaskPanel.jsx
Wyświetl plik

@ -3,6 +3,7 @@ import React from 'react';
import EditTaskForm from './EditTaskForm';
import PropTypes from 'prop-types';
import Storage from '../classes/Storage';
import ResizeModes from '../classes/ResizeModes';
class NewTaskPanel extends React.Component {
static defaultProps = {
@ -25,14 +26,14 @@ class NewTaskPanel extends React.Component {
this.state = {
name: props.name,
editTaskFormLoaded: false,
resize: Storage.getItem('do_resize') !== null ? Storage.getItem('do_resize') == "1" : true,
resizeMode: Storage.getItem('resize_mode') === null ? ResizeModes.YES : ResizeModes.fromString(Storage.getItem('resize_mode')),
resizeSize: parseInt(Storage.getItem('resize_size')) || 2048
};
this.save = this.save.bind(this);
this.handleFormTaskLoaded = this.handleFormTaskLoaded.bind(this);
this.getTaskInfo = this.getTaskInfo.bind(this);
this.setResize = this.setResize.bind(this);
this.setResizeMode = this.setResizeMode.bind(this);
this.handleResizeSizeChange = this.handleResizeSizeChange.bind(this);
}
@ -40,7 +41,7 @@ class NewTaskPanel extends React.Component {
e.preventDefault();
this.taskForm.saveLastPresetToStorage();
Storage.setItem('resize_size', this.state.resizeSize);
Storage.setItem('do_resize', this.state.resize ? "1" : "0");
Storage.setItem('resize_mode', this.state.resizeMode);
if (this.props.onSave) this.props.onSave(this.getTaskInfo());
}
@ -54,13 +55,14 @@ class NewTaskPanel extends React.Component {
getTaskInfo(){
return Object.assign(this.taskForm.getTaskInfo(), {
resizeTo: (this.state.resize && this.state.resizeSize > 0) ? this.state.resizeSize : null
resizeSize: this.state.resizeSize,
resizeMode: this.state.resizeMode
});
}
setResize(flag){
setResizeMode(v){
return e => {
this.setState({resize: flag});
this.setState({resizeMode: v});
}
}
@ -91,23 +93,19 @@ class NewTaskPanel extends React.Component {
<div className="col-sm-10">
<div className="btn-group">
<button type="button" className="btn btn-default dropdown-toggle" data-toggle="dropdown">
{this.state.resize ?
"Yes" : "No"} <span className="caret"></span>
{ResizeModes.toHuman(this.state.resizeMode)} <span className="caret"></span>
</button>
<ul className="dropdown-menu">
<li>
<a href="javascript:void(0);"
onClick={this.setResize(true)}>
<i style={{opacity: this.state.resize ? 1 : 0}} className="fa fa-check"></i> Yes</a>
</li>
<li>
<a href="javascript:void(0);"
onClick={this.setResize(false)}>
<i style={{opacity: !this.state.resize ? 1 : 0}} className="fa fa-check"></i> Skip</a>
</li>
{ResizeModes.all().map(mode =>
<li key={mode}>
<a href="javascript:void(0);"
onClick={this.setResizeMode(mode)}>
<i style={{opacity: this.state.resizeMode === mode ? 1 : 0}} className="fa fa-check"></i> {ResizeModes.toHuman(mode)}</a>
</li>
)}
</ul>
</div>
<div className={"resize-control " + (!this.state.resize ? "hide" : "")}>
<div className={"resize-control " + (this.state.resizeMode === ResizeModes.NO ? "hide" : "")}>
<input
type="number"
step="100"

9
app/static/app/js/components/ProjectListItem.jsx
Wyświetl plik

@ -11,6 +11,7 @@ import Dropzone from '../vendor/dropzone';
import csrf from '../django/csrf';
import HistoryNav from '../classes/HistoryNav';
import PropTypes from 'prop-types';
import ResizeModes from '../classes/ResizeModes';
import $ from 'jquery';
class ProjectListItem extends React.Component {
@ -180,6 +181,10 @@ class ProjectListItem extends React.Component {
if (!formData.has || !formData.has("options")) formData.append("options", JSON.stringify(taskInfo.options));
if (!formData.has || !formData.has("processing_node")) formData.append("processing_node", taskInfo.selectedNode.id);
if (!formData.has || !formData.has("auto_processing_node")) formData.append("auto_processing_node", taskInfo.selectedNode.key == "auto");
if (taskInfo.resizeMode === ResizeModes.YES){
if (!formData.has || !formData.has("resize_to")) formData.append("resize_to", taskInfo.resizeSize);
}
});
}
}
@ -225,8 +230,8 @@ class ProjectListItem extends React.Component {
this.dz._taskInfo = taskInfo; // Allow us to access the task info from dz
// Update dropzone settings
if (taskInfo.resizeTo !== null){
this.dz.options.resizeWidth = taskInfo.resizeTo;
if (taskInfo.resizeMode === ResizeModes.YESINBROWSER){
this.dz.options.resizeWidth = taskInfo.resizeSize;
this.dz.options.resizeQuality = 1.0;
this.setUploadState({resizing: true, editing: false});

2
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -351,7 +351,7 @@ class TaskListItem extends React.Component {
let status = statusCodes.description(task.status);
if (status === "") status = "Uploading images";
if (!task.processing_node) status = "";
if (!task.processing_node) status = "Waiting for a node...";
if (task.pending_action !== null) status = pendingActions.description(task.pending_action);
let expanded = "";

2
start.sh
Wyświetl plik

@ -69,6 +69,8 @@ if [ "$WO_SSL" = "YES" ]; then
proto="https"
fi
./worker.sh scheduler start
congrats(){
(sleep 5; echo
echo -e "\033[92m"

2
webodm/settings.py
Wyświetl plik

@ -322,6 +322,8 @@ CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_INCLUDE=['worker.tasks']
CELERY_WORKER_REDIRECT_STDOUTS = False
CELERY_WORKER_HIJACK_ROOT_LOGGER = False
if TESTING:
CELERY_TASK_ALWAYS_EAGER = True

35
worker.sh
Wyświetl plik

@ -9,7 +9,9 @@ usage(){
echo "This program manages the background worker processes. WebODM requires at least one background process worker to be running at all times."
echo
echo "Command list:"
echo " start Start background worker"
echo " start Start background worker"
echo " scheduler start Start background worker scheduler"
echo " scheduler stop Stop background worker scheduler"
exit
}
@ -51,12 +53,41 @@ start(){
action=$1
echo "Starting worker using broker at $WO_BROKER"
celery -A worker worker --loglevel=info
celery -A worker worker --loglevel=warn > /dev/null
}
start_scheduler(){
stop_scheduler
if [[ ! -f ./celerybeat.pid ]]; then
celery -A worker beat &
else
echo "Scheduler already running (celerybeat.pid exists)."
fi
}
stop_scheduler(){
if [[ -f ./celerybeat.pid ]]; then
kill -9 $(cat ./celerybeat.pid) 2>/dev/null
rm ./celerybeat.pid 2>/dev/null
echo "Scheduler has shutdown."
else
echo "Scheduler is not running."
fi
}
if [[ $1 = "start" ]]; then
environment_check
start
elif [[ $1 = "scheduler" ]]; then
if [[ $2 = "start" ]]; then
environment_check
start_scheduler
elif [[ $2 = "stop" ]]; then
environment_check
stop_scheduler
else
usage
fi
else
usage
fi

67
worker/tasks.py
Wyświetl plik

@ -1,24 +1,17 @@
import os
import traceback
import re
import piexif
from celery.utils.log import get_task_logger
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction
from django.db.models import Count
from django.db.models import Q
from app.models import Project
from app.models import Task
from webodm import settings
from nodeodm import status_codes
from nodeodm.models import ProcessingNode
from webodm import settings
from .celery import app
from celery.utils.log import get_task_logger
from django.db import transaction
from PIL import Image
from functools import partial
from multiprocessing import Pool, cpu_count
logger = get_task_logger(__name__)
@ -84,57 +77,3 @@ def process_pending_tasks():
for task in tasks:
process_task.delay(task.id)
@app.task
def resize_image(image_path, resize_to):
try:
exif_dict = piexif.load(image_path)
im = Image.open(image_path)
path, ext = os.path.splitext(image_path)
resized_image_path = os.path.join(path + '.resized' + ext)
width, height = im.size
max_side = max(width, height)
if max_side < resize_to:
logger.warning('We are making {} bigger ({} --> {})'.format(image_path, max_side, resize_to))
ratio = float(resize_to) / float(max_side)
resized_width = int(width * ratio)
resized_height = int(height * ratio)
im.thumbnail((resized_width, resized_height), Image.LANCZOS)
if len(exif_dict['Exif']) > 0:
exif_dict['Exif'][piexif.ExifIFD.PixelXDimension] = resized_width
exif_dict['Exif'][piexif.ExifIFD.PixelYDimension] = resized_height
im.save(resized_image_path, "JPEG", exif=piexif.dump(exif_dict), quality=100)
else:
im.save(resized_image_path, "JPEG", quality=100)
im.close()
# Delete original image, rename resized image to original
os.remove(image_path)
os.rename(resized_image_path, image_path)
logger.info("Resized {}".format(os.path.basename(resized_image_path)))
except IOError as e:
logger.warning("Cannot resize {}: {}.".format(image_path, str(e)))
return None
return image_path
@app.task
def resize_images(directory, resize_to):
"""
Destructively resize a directory of JPG images while retaining EXIF tags.
Resulting images are always converted to JPG.
TODO: add support for tiff files
:return list containing paths of resized images
"""
images_path = [os.path.join(directory, f) for f in os.listdir(directory) if re.match(r'.*\.jpe?g$', f, re.IGNORECASE)]
resized_images = list(filter(lambda i: i is not None, Pool(cpu_count()).map(
partial(resize_image, resize_to=resize_to),
images_path)))
return resized_images