diff --git a/app/api/processingnodes.py b/app/api/processingnodes.py index 0b1ea188..ed37c384 100644 --- a/app/api/processingnodes.py +++ b/app/api/processingnodes.py @@ -34,7 +34,7 @@ class ProcessingNodeFilter(FilterSet): class Meta: model = ProcessingNode - fields = ['has_available_options', 'id', 'hostname', 'port', 'api_version', 'queue_count', 'max_images', 'label', ] + fields = ['has_available_options', 'id', 'hostname', 'port', 'api_version', 'queue_count', 'max_images', 'label', 'engine', 'engine_version', ] class ProcessingNodeViewSet(viewsets.ModelViewSet): """ diff --git a/app/boot.py b/app/boot.py index e2d80352..e8a5cb6f 100644 --- a/app/boot.py +++ b/app/boot.py @@ -97,11 +97,9 @@ def boot(): def add_default_presets(): try: Preset.objects.update_or_create(name='Volume Analysis', system=True, - defaults={'options': [{'name': 'use-opensfm-dense', 'value': True}, - {'name': 'dsm', 'value': True}, + defaults={'options': [{'name': 'dsm', 'value': True}, {'name': 'dem-resolution', 'value': '2'}, - {'name': 'depthmap-resolution', 'value': '1000'}, - {'name': 'opensfm-depthmap-min-patch-sd', 'value': '0'}]}) + {'name': 'depthmap-resolution', 'value': '1000'}]}) Preset.objects.update_or_create(name='3D Model', system=True, defaults={'options': [{'name': 'mesh-octree-depth', 'value': "11"}, {'name': 'use-3dmesh', 'value': True}, @@ -126,7 +124,8 @@ def add_default_presets(): Preset.objects.update_or_create(name='Fast Orthophoto', system=True, defaults={'options': [{'name': 'fast-orthophoto', 'value': True}]}) Preset.objects.update_or_create(name='High Resolution', system=True, - defaults={'options': [{'name': 'dsm', 'value': True}, + defaults={'options': [{'name': 'ignore-gsd', 'value': True}, + {'name': 'dsm', 'value': True}, {'name': 'depthmap-resolution', 'value': '1000'}, {'name': 'dem-resolution', 'value': "2.0"}, {'name': 'orthophoto-resolution', 'value': "2.0"}, diff --git a/app/models/task.py b/app/models/task.py index cf31db68..877c6576 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -22,6 +22,7 @@ from django.contrib.postgres import fields from django.core.exceptions import ValidationError from django.db import models from django.db import transaction +from django.db import connection from django.utils import timezone from urllib3.exceptions import ReadTimeoutError @@ -115,7 +116,7 @@ def resize_image(image_path, resize_to, done=None): os.rename(resized_image_path, image_path) logger.info("Resized {} to {}x{}".format(image_path, resized_width, resized_height)) - except IOError as e: + except (IOError, ValueError) as e: logger.warning("Cannot resize {}: {}.".format(image_path, str(e))) if done is not None: done() @@ -174,22 +175,7 @@ class Task(models.Model): (pending_actions.IMPORT, 'IMPORT'), ) - # Not an exact science - TASK_OUTPUT_MILESTONES_LAST_VALUE = 0.85 - TASK_OUTPUT_MILESTONES = { - 'Running ODM Load Dataset Cell': 0.01, - 'Running ODM Load Dataset Cell - Finished': 0.05, - 'opensfm/bin/opensfm match_features': 0.10, - 'opensfm/bin/opensfm reconstruct': 0.20, - 'opensfm/bin/opensfm export_visualsfm': 0.30, - 'Running ODM Meshing Cell': 0.50, - 'Running MVS Texturing Cell': 0.55, - 'Running ODM Georeferencing Cell': 0.60, - 'Running ODM DEM Cell': 0.70, - 'Running ODM Orthophoto Cell': 0.75, - 'Running ODM OrthoPhoto Cell - Finished': 0.80, - 'Compressing all.zip:': TASK_OUTPUT_MILESTONES_LAST_VALUE - } + TASK_PROGRESS_LAST_VALUE = 0.85 id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False) @@ -373,7 +359,11 @@ class Task(models.Model): raise NodeServerError(e) self.refresh_from_db() - self.extract_assets_and_complete() + + try: + self.extract_assets_and_complete() + except zipfile.BadZipFile: + raise NodeServerError("Invalid zip file") images_json = self.assets_path("images.json") if os.path.exists(images_json): @@ -573,22 +563,18 @@ class Task(models.Model): # Need to update status (first time, queued or running?) if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]: # Update task info from processing node - info = self.processing_node.get_task_info(self.uuid) + current_lines_count = len(self.console_output.split("\n")) + + info = self.processing_node.get_task_info(self.uuid, current_lines_count) self.processing_time = info.processing_time self.status = info.status.value - current_lines_count = len(self.console_output.split("\n")) - console_output = self.processing_node.get_task_console_output(self.uuid, current_lines_count) - if len(console_output) > 0: - self.console_output += "\n".join(console_output) + '\n' + if len(info.output) > 0: + self.console_output += "\n".join(info.output) + '\n' # Update running progress - for line in console_output: - for line_match, value in self.TASK_OUTPUT_MILESTONES.items(): - if line_match in line: - self.running_progress = value - break + self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE if info.last_error != "": self.last_error = info.last_error @@ -607,9 +593,10 @@ class Task(models.Model): os.makedirs(assets_dir) - logger.info("Downloading all.zip for {}".format(self)) - - # Download all assets + # Download and try to extract results up to 4 times + # (~95% of the times, on large downloads, the archive could be corrupted) + retry_num = 0 + extracted = False last_update = 0 def callback(progress): @@ -619,17 +606,32 @@ class Task(models.Model): if time_has_elapsed or int(progress) == 100: Task.objects.filter(pk=self.id).update(running_progress=( - self.TASK_OUTPUT_MILESTONES_LAST_VALUE + (float(progress) / 100.0) * 0.1)) + self.TASK_PROGRESS_LAST_VALUE + (float(progress) / 100.0) * 0.1)) last_update = time.time() - zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback) + while not extracted: + last_update = 0 + logger.info("Downloading all.zip for {}".format(self)) - # Rename to all.zip - os.rename(zip_path, os.path.join(os.path.dirname(zip_path), 'all.zip')) + # Download all assets + zip_path = self.processing_node.download_task_assets(self.uuid, assets_dir, progress_callback=callback) - logger.info("Extracting all.zip for {}".format(self)) + # Rename to all.zip + all_zip_path = self.assets_path("all.zip") + os.rename(zip_path, all_zip_path) - self.extract_assets_and_complete() + logger.info("Extracting all.zip for {}".format(self)) + + try: + self.extract_assets_and_complete() + extracted = True + except zipfile.BadZipFile: + if retry_num < 4: + logger.warning("{} seems corrupted. Retrying...".format(all_zip_path)) + retry_num += 1 + os.remove(all_zip_path) + else: + raise NodeServerError("Invalid zip file") else: # FAILED, CANCELED self.save() @@ -648,17 +650,15 @@ class Task(models.Model): def extract_assets_and_complete(self): """ Extracts assets/all.zip and populates task fields where required. + It will raise a zipfile.BadZipFile exception is the archive is corrupted. :return: """ assets_dir = self.assets_path("") zip_path = self.assets_path("all.zip") # Extract from zip - try: - with zipfile.ZipFile(zip_path, "r") as zip_h: - zip_h.extractall(assets_dir) - except zipfile.BadZipFile: - raise NodeServerError("Invalid zip file") + with zipfile.ZipFile(zip_path, "r") as zip_h: + zip_h.extractall(assets_dir) logger.info("Extracted all.zip for {}".format(self)) @@ -678,6 +678,12 @@ class Task(models.Model): raster = GDALRaster(raster_path) extent = OGRGeometry.from_bbox(raster.extent) + # Make sure PostGIS supports it + with connection.cursor() as cursor: + cursor.execute("SELECT SRID FROM spatial_ref_sys WHERE SRID = %s", [raster.srid]) + if cursor.rowcount == 0: + raise NodeServerError("Unsupported SRS {}. Please make sure you picked a supported SRS.".format(raster.srid)) + # It will be implicitly transformed into the SRID of the model’s field # self.field = GEOSGeometry(...) setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) diff --git a/app/static/app/js/Console.jsx b/app/static/app/js/Console.jsx index 99aadde3..c3911c93 100644 --- a/app/static/app/js/Console.jsx +++ b/app/static/app/js/Console.jsx @@ -20,12 +20,15 @@ class Console extends React.Component { } this.autoscroll = props.autoscroll === true; + this.showFullscreenButton = props.showFullscreenButton === true; this.setRef = this.setRef.bind(this); this.handleMouseOver = this.handleMouseOver.bind(this); this.handleMouseOut = this.handleMouseOut.bind(this); this.downloadTxt = this.downloadTxt.bind(this); this.copyTxt = this.copyTxt.bind(this); + this.enterFullscreen = this.enterFullscreen.bind(this); + this.exitFullscreen = this.exitFullscreen.bind(this); } componentDidMount(){ @@ -79,6 +82,19 @@ class Console extends React.Component { console.log("Output copied to clipboard"); } + enterFullscreen(){ + const consoleElem = this.$console.get(0); + if (consoleElem.requestFullscreen) { + consoleElem.requestFullscreen(); + } + } + + exitFullscreen(){ + if (document.exitFullscreen){ + document.exitFullscreen(); + } + } + tearDownDynamicSource(){ if (this.sourceTimeout) clearTimeout(this.sourceTimeout); if (this.sourceRequest) this.sourceRequest.abort(); @@ -138,11 +154,17 @@ class Console extends React.Component { onMouseOver={this.handleMouseOver} onMouseOut={this.handleMouseOut} ref={this.setRef} - >{lines.map(line => { + > + Exit Fullscreen + + {lines.map(line => { if (this.props.lang) return (
); else return line + "\n"; })} {"\n"} + + Exit Fullscreen + ]; if (this.props.showConsoleButtons){ @@ -153,6 +175,9 @@ class Console extends React.Component { + + + ); } diff --git a/app/static/app/js/components/BasicTaskView.jsx b/app/static/app/js/components/BasicTaskView.jsx deleted file mode 100644 index a5ce0d4f..00000000 --- a/app/static/app/js/components/BasicTaskView.jsx +++ /dev/null @@ -1,235 +0,0 @@ -import React from 'react'; -import PropTypes from 'prop-types'; -import '../css/BasicTaskView.scss'; -import update from 'immutability-helper'; -import PipelineSteps from '../classes/PipelineSteps'; -import StatusCodes from '../classes/StatusCodes'; -import $ from 'jquery'; - -class BasicTaskView extends React.Component { - static defaultProps = {}; - - static propTypes = { - source: PropTypes.oneOfType([ - PropTypes.string.isRequired, - PropTypes.func.isRequired - ]), - taskStatus: PropTypes.number, - onAddLines: PropTypes.func - }; - - constructor(props){ - super(); - - this.state = { - lines: [], - currentRf: 0, - rf: PipelineSteps.get(), - loaded: false - }; - - this.imageUpload = { - action: "imageupload", - label: "Image Resize / Upload", - icon: "fa fa-image" - }; - - // Add a post processing step - this.state.rf.push({ - action: "postprocessing", - label: "Post Processing", - icon: "fa fa-file-archive-o", - beginsWith: "Running ODM OrthoPhoto Cell - Finished", - endsWith: null - }); - - this.addLines = this.addLines.bind(this); - this.setupDynamicSource = this.setupDynamicSource.bind(this); - this.reset = this.reset.bind(this); - this.tearDownDynamicSource = this.tearDownDynamicSource.bind(this); - this.getRfEndStatus = this.getRfEndStatus.bind(this); - this.getRfRunningStatus = this.getRfRunningStatus.bind(this); - this.suffixFor = this.suffixFor.bind(this); - this.updateRfState = this.updateRfState.bind(this); - this.getInitialStatus = this.getInitialStatus.bind(this); - } - - componentDidMount(){ - this.reset(); - } - - setupDynamicSource(){ - const updateFromSource = () => { - let sourceUrl = typeof this.props.source === 'function' ? - this.props.source(this.state.lines.length) : - this.props.source; - - // Fetch - this.sourceRequest = $.get(sourceUrl, text => { - if (text !== ""){ - let lines = text.split("\n"); - this.addLines(lines); - } - }) - .always((_, textStatus) => { - if (textStatus !== "abort" && this.props.refreshInterval !== undefined){ - this.sourceTimeout = setTimeout(updateFromSource, this.props.refreshInterval); - } - if (!this.state.loaded) this.setState({loaded: true}); - }); - }; - - updateFromSource(); - } - - getRfEndStatus(){ - return this.props.taskStatus === StatusCodes.COMPLETED ? - 'completed' : - 'errored'; - } - - getRfRunningStatus(){ - return [StatusCodes.RUNNING, StatusCodes.QUEUED].indexOf(this.props.taskStatus) !== -1 ? - 'running' : - 'errored'; - } - - getInitialStatus(){ - if ([null, StatusCodes.QUEUED, StatusCodes.RUNNING].indexOf(this.props.taskStatus) !== -1){ - return 'queued'; - }else{ - return this.getRfEndStatus(); - } - } - - reset(){ - this.state.rf.forEach(p => { - p.state = this.getInitialStatus(); - }); - - if ([StatusCodes.RUNNING].indexOf(this.props.taskStatus) !== -1){ - this.state.rf[0].state = 'running'; - } - - this.tearDownDynamicSource(); - this.setState({lines: [], currentRf: 0, loaded: false, rf: this.state.rf}); - this.setupDynamicSource(); - } - - tearDownDynamicSource(){ - if (this.sourceTimeout) clearTimeout(this.sourceTimeout); - if (this.sourceRequest) this.sourceRequest.abort(); - } - - componentDidUpdate(prevProps){ - - let taskFailed; - let taskCompleted; - let taskRestarted; - - taskFailed = [StatusCodes.FAILED, StatusCodes.CANCELED].indexOf(this.props.taskStatus) !== -1; - taskCompleted = this.props.taskStatus === StatusCodes.COMPLETED; - - if (prevProps.taskStatus !== this.props.taskStatus){ - taskRestarted = this.props.taskStatus === null; - } - - this.updateRfState(taskFailed, taskCompleted, taskRestarted); - } - - componentWillUnmount(){ - this.tearDownDynamicSource(); - } - - addLines(lines){ - if (!Array.isArray(lines)) lines = [lines]; - - let currentRf = this.state.currentRf; - - const updateRf = (rfIndex, line) => { - const current = this.state.rf[rfIndex]; - if (!current) return; - - if (current.beginsWith && line.endsWith(current.beginsWith)){ - current.state = this.getRfRunningStatus(); - - // Set previous as done - if (this.state.rf[rfIndex - 1]){ - this.state.rf[rfIndex - 1].state = 'completed'; - } - }else if (current.endsWith && line.endsWith(current.endsWith)){ - current.state = this.getRfEndStatus(); - - // Check next - updateRf(rfIndex + 1, line); - - currentRf = rfIndex + 1; - } - }; - - lines.forEach(line => { - updateRf(currentRf, line); - }); - - this.setState(update(this.state, { - lines: {$push: lines} - })); - this.setState({ - rf: this.state.rf, - currentRf - }); - this.updateRfState(); - - if (this.props.onAddLines) this.props.onAddLines(lines); - } - - updateRfState(taskFailed, taskCompleted, taskRestarted){ - // If the task has just failed, update all items that were either running or in queued state - if (taskFailed){ - this.state.rf.forEach(p => { - if (p.state === 'queued' || p.state === 'running') p.state = this.getInitialStatus(); - }); - } - - // If completed, all steps must have completed - if (taskCompleted){ - this.state.rf.forEach(p => p.state = 'completed'); - } - - if (taskRestarted){ - this.state.rf.forEach(p => p.state = 'queued'); - } - } - - suffixFor(state){ - if (state === 'running'){ - return (...); - }else if (state === 'completed'){ - return (); - }else if (state === 'errored'){ - return (); - } - } - - render() { - const { rf, loaded } = this.state; - const imageUploadState = this.props.taskStatus === null - ? 'running' - : 'completed'; - - return (