Contours GRASS/GDAL script working, async execution, grass engine changes, auto cleanup flag, contours UI

pull/639/head
Piero Toffanin 2019-03-30 19:07:24 -04:00
rodzic 04c535653c
commit b46163ff93
18 zmienionych plików z 359 dodań i 134 usunięć

Wyświetl plik

@ -28,13 +28,14 @@ class GrassEngine:
class GrassContext:
def __init__(self, grass_binary, tmpdir = None, template_args = {}, location = None):
def __init__(self, grass_binary, tmpdir = None, template_args = {}, location = None, auto_cleanup=True):
self.grass_binary = grass_binary
if tmpdir is None:
tmpdir = os.path.basename(tempfile.mkdtemp('_grass_engine', dir=settings.MEDIA_TMP))
self.tmpdir = tmpdir
self.template_args = template_args
self.location = location
self.auto_cleanup = auto_cleanup
def get_cwd(self):
return os.path.join(settings.MEDIA_TMP, self.tmpdir)
@ -82,6 +83,9 @@ class GrassContext:
tmpl = Template(script_content)
# Write script to disk
if not os.path.exists(self.get_cwd()):
os.mkdir(self.get_cwd())
with open(os.path.join(self.get_cwd(), 'script.sh'), 'w') as f:
f.write(tmpl.substitute(self.template_args))
@ -94,6 +98,9 @@ class GrassContext:
out = out.decode('utf-8').strip()
err = err.decode('utf-8').strip()
logger.info("GOT!")
logger.info(out)
if p.returncode == 0:
return out
else:
@ -103,15 +110,18 @@ class GrassContext:
return {
'tmpdir': self.tmpdir,
'template_args': self.template_args,
'location': self.location
'location': self.location,
'auto_cleanup': self.auto_cleanup
}
def __del__(self):
pass
# Cleanup
def cleanup(self):
if os.path.exists(self.get_cwd()):
shutil.rmtree(self.get_cwd())
def __del__(self):
if self.auto_cleanup:
self.cleanup()
class GrassEngineException(Exception):
pass

Wyświetl plik

@ -113,11 +113,13 @@ class TestPlugins(BootTestCase):
}""")
ctx.set_location("EPSG:4326")
output = execute_grass_script.delay(
result = execute_grass_script.delay(
os.path.join(grass_scripts_dir, "simple_test.grass"),
ctx.serialize()
).get()
self.assertTrue("Number of points: 1" in output)
self.assertTrue("Number of points: 1" in result.get('output'))
self.assertTrue(result.get('context') == ctx.serialize())
error = execute_grass_script.delay(
os.path.join(grass_scripts_dir, "nonexistant_script.grass"),
@ -129,6 +131,7 @@ class TestPlugins(BootTestCase):
with self.assertRaises(GrassEngineException):
ctx.execute(os.path.join(grass_scripts_dir, "nonexistant_script.grass"))
# TODO: verify autocleanup works
def test_plugin_datastore(self):
test_plugin = get_plugin_by_name("test")

Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "WebODM",
"version": "0.8.2",
"version": "0.9.0",
"description": "Open Source Drone Image Processing",
"main": "index.js",
"scripts": {

Wyświetl plik

@ -0,0 +1,72 @@
import os
from rest_framework import status
from rest_framework.response import Response
from app.plugins.views import TaskView
from worker.tasks import execute_grass_script
from app.plugins.grass_engine import grass, GrassEngineException
from worker.celery import app as celery
class TaskContoursGenerate(TaskView):
def post(self, request, pk=None):
task = self.get_and_check_task(request, pk)
layer = request.data.get('layer', None)
if layer == 'DSM' and task.dsm_extent is None:
return Response({'error': 'No DSM layer is available.'})
elif layer == 'DTM' and task.dtm_extent is None:
return Response({'error': 'No DTM layer is available.'})
try:
if layer == 'DSM':
dem = os.path.abspath(task.get_asset_download_path("dsm.tif"))
elif layer == 'DTM':
dem = os.path.abspath(task.get_asset_download_path("dtm.tif"))
else:
raise GrassEngineException('{} is not a valid layer.'.format(layer))
context = grass.create_context({'auto_cleanup' : False})
epsg = int(request.data.get('epsg', '3857'))
interval = float(request.data.get('interval', 1))
format = request.data.get('format', 'GPKG')
supported_formats = ['GPKG', 'ESRI Shapefile', 'DXF', 'GeoJSON']
if not format in supported_formats:
raise GrassEngineException("Invalid format {} (must be one of: {})".format(format, ",".join(supported_formats)))
simplify = float(request.data.get('simplify', 0.01))
context.add_param('dem_file', dem)
context.add_param('interval', interval)
context.add_param('format', format)
context.add_param('simplify', simplify)
context.add_param('epsg', epsg)
context.set_location('epsg:' + str(epsg))
celery_task_id = execute_grass_script.delay(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"calc_contours.grass"
), context.serialize()).task_id
return Response({'celery_task_id': celery_task_id}, status=status.HTTP_200_OK)
except GrassEngineException as e:
return Response({'error': str(e)}, status=status.HTTP_200_OK)
class TaskContoursCheck(TaskView):
def get(self, request, pk=None, celery_task_id=None):
task = self.get_and_check_task(request, pk)
# res = celery.AsyncResult(celery_task_id)
# res.wait()
# print(res.get())
#while not res.ready():
#if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error'])
# if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error'])
#
# cols = output.split(':')
# if len(cols) == 7:
# return Response({'volume': str(abs(float(cols[6])))}, status=status.HTTP_200_OK)
# else:
# raise GrassEngineException(output)

Wyświetl plik

@ -0,0 +1,38 @@
# dem_file: GeoTIFF DEM containing the surface to calculate contours
# interval: Contours interval
# format: OGR output format
# simplify: Simplify value
# epsg: target EPSG code
# destination: destination folder. If it does not exist, it will be created.
#
# ------
# output: If successful, prints the full path to the contours file. Otherwise it prints "error"
ext=""
if [ "${format}" = "GeoJSON" ]; then
ext="geojson"
elif [ "${format}" = "GPKG" ]; then
ext="gpkg"
elif [ "${format}" = "DXF" ]; then
ext="dxf"
elif [ "${format}" = "ESRI Shapefile" ]; then
ext="shp"
fi
gdal_contour -i ${interval} -f GPKG "${dem_file}" contours.gpkg > /dev/null
ogr2ogr -simplify ${simplify} -t_srs EPSG:${epsg} -overwrite -f "${format}" output.$$ext contours.gpkg > /dev/null
if [ -e "output.$$ext" ]; then
# ESRI ShapeFile extra steps to compress into a zip archive
# we leverage Python's shutil in this case
if [ "${format}" = "ESRI Shapefile" ]; then
ext="zip"
mkdir contours/
mv output* contours/
echo "import shutil;shutil.make_archive('output', 'zip', 'contours/')" | python
fi
echo "$$(pwd)/output.$$ext"
else
echo "error"
fi

Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "Contours",
"webodmMinVersion": "0.8.2",
"webodmMinVersion": "0.9.0",
"description": "Compute, preview and export contours from DEMs",
"version": "1.0.0",
"author": "Piero Toffanin",

Wyświetl plik

@ -1,8 +1,18 @@
from app.plugins import PluginBase
from app.plugins import MountPoint
from .api import TaskContoursGenerate
from .api import TaskContoursCheck
class Plugin(PluginBase):
def include_js_files(self):
return ['main.js']
def build_jsx_components(self):
return ['Contours.jsx']
return ['Contours.jsx']
def api_mount_points(self):
return [
MountPoint('task/(?P<pk>[^/.]+)/contours/generate', TaskContoursGenerate.as_view()),
MountPoint('task/(?P<pk>[^/.]+)/contours/check/(?P<celery_task_id>.+)', TaskContoursCheck.as_view()),
]

Wyświetl plik

@ -1,10 +1,15 @@
import L from 'leaflet';
import ReactDOM from 'ReactDOM';
import React from 'react';
import PropTypes from 'prop-types';
import './Contours.scss';
import ContoursPanel from './ContoursPanel';
class ContoursButton extends React.Component {
static propTypes = {
tasks: PropTypes.object.isRequired
}
constructor(props){
super(props);
@ -28,7 +33,7 @@ class ContoursButton extends React.Component {
<a href="javascript:void(0);"
onClick={this.handleOpen}
className="leaflet-control-contours-button leaflet-bar-part theme-secondary"></a>
<ContoursPanel onClose={this.handleClose} />
<ContoursPanel isShowed={showPanel} tasks={this.props.tasks} onClose={this.handleClose} />
</div>);
}
}
@ -41,9 +46,8 @@ export default L.Control.extend({
onAdd: function (map) {
var container = L.DomUtil.create('div', 'leaflet-control-contours leaflet-bar leaflet-control');
L.DomEvent.disableClickPropagation(container);
ReactDOM.render(<ContoursButton />, container);
ReactDOM.render(<ContoursButton tasks={this.options.tasks} />, container);
// this._map = map;
return container;
}
});

Wyświetl plik

@ -1,4 +1,6 @@
.leaflet-control-contours{
z-index: 999;
a.leaflet-control-contours-button{
background: url(icon.svg) no-repeat 0 0;
background-size: 26px 26px;

Wyświetl plik

@ -3,13 +3,15 @@ import PropTypes from 'prop-types';
import Storage from 'webodm/classes/Storage';
import L from 'leaflet';
import './ContoursPanel.scss';
import ErrorMessage from 'webodm/components/ErrorMessage';
export default class ContoursPanel extends React.Component {
static defaultProps = {
};
static propTypes = {
onClose: PropTypes.func.isRequired
onClose: PropTypes.func.isRequired,
tasks: PropTypes.object.isRequired,
isShowed: PropTypes.bool.isRequired
}
constructor(props){
@ -17,37 +19,52 @@ export default class ContoursPanel extends React.Component {
this.state = {
error: "",
permanentError: "",
interval: Storage.getItem("last_contours_interval") || "1",
customInterval: Storage.getItem("last_contours_custom_interval") || "1",
layer: "",
projection: Storage.getItem("last_contours_projection") || "4326",
customProjection: Storage.getItem("last_contours_custom_projection") || "4326",
epsg: Storage.getItem("last_contours_epsg") || "4326",
customEpsg: Storage.getItem("last_contours_custom_epsg") || "4326",
layers: [],
loading: true,
task: props.tasks[0] || null,
previewLoading: false,
};
}
componentDidMount(){
componentDidUpdate(){
if (this.props.isShowed && this.state.loading){
const {id, project} = this.state.task;
this.loadingReq = $.getJSON(`/api/projects/${project}/tasks/${id}/`)
.done(res => {
const { available_assets } = res;
let layers = [];
if (available_assets.indexOf("dsm.tif") !== -1) layers.push("DSM");
if (available_assets.indexOf("dtm.tif") !== -1) layers.push("DTM");
if (layers.length > 0){
this.setState({layers, layer: layers[0]});
}else{
this.setState({permanentError: "No DSM or DTM is available. To export contours, make sure to process a task with either the --dsm or --dtm option checked."});
}
})
.fail(() => {
this.setState({permanentError: `Cannot retrieve information for task ${id}. Are you are connected to the internet.`})
})
.always(() => {
this.setState({loading: false});
this.loadingReq = null;
});
}
}
componentWillUnmount(){
}
calculateVolume(){
// $.ajax({
// type: 'POST',
// url: `/api/plugins/measure/task/${task.id}/volume`,
// data: JSON.stringify({'area': this.props.resultFeature.toGeoJSON()}),
// contentType: "application/json"
// }).done(result => {
// if (result.volume){
// this.setState({volume: parseFloat(result.volume)});
// }else if (result.error){
// this.setState({error: result.error});
// }else{
// this.setState({error: "Invalid response: " + result});
// }
// }).fail(error => {
// this.setState({error});
// });
if (this.loadingReq){
this.loadingReq.abort();
this.loadingReq = null;
}
}
handleSelectInterval = e => {
@ -62,95 +79,150 @@ export default class ContoursPanel extends React.Component {
this.setState({customInterval: e.target.value});
}
handleSelectProjection = e => {
this.setState({projection: e.target.value});
handleSelectEpsg = e => {
this.setState({Epsg: e.target.value});
}
handleChangeCustomProjection = e => {
this.setState({customProjection: e.target.value});
handleChangeCustomEpsg = e => {
this.setState({customEpsg: e.target.value});
}
getFormValues = () => {
const { interval, customInterval, epsg, customEpsg, layer } = this.state;
return {
interval: interval !== "custom" ? interval : customInterval,
epsg: epsg !== "custom" ? epsg : customEpsg,
layer
};
}
handleShowPreview = () => {
this.setState({previewLoading: true});
const data = this.getFormValues();
data.interval = 1;
data.epsg = 3857;
data.format = "GeoJSON";
data.simplify = 0.05;
$.ajax({
type: 'POST',
url: `/api/plugins/contours/task/${this.state.task.id}/contours/generate`,
data: data
}).done(result => {
if (result.celery_task_id){
console.log(result);
}else if (result.error){
this.setState({error: result.error});
}else{
this.setState({error: "Invalid response: " + result});
}
this.setState({previewLoading: false});
}).fail(error => {
this.setState({previewLoading: false, error: JSON.stringify(error)});
});
}
render(){
const { error, interval, customInterval, layer,
projection, customProjection } = this.state;
const { loading, task, layers, error, permanentError, interval, customInterval, layer,
epsg, customEpsg,
previewLoading } = this.state;
const intervalValues = [0.25, 0.5, 1, 1.5, 2];
const disabled = (interval === "custom" && !customInterval) ||
(epsg === "custom" && !customEpsg);
let content = "";
if (loading) content = (<span><i className="fa fa-circle-o-notch fa-spin"></i> Loading...</span>);
else if (error) content = (<ErrorMessage bind={[this, "error"]} />);
else if (permanentError) content = (<div className="alert alert-warning">{permanentError}</div>);
else{
content = (<div>
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Interval:</label>
<div className="col-sm-9 ">
<select className="form-control" value={interval} onChange={this.handleSelectInterval}>
{intervalValues.map(iv => <option value={iv}>{iv} meter</option>)}
<option value="custom">Custom</option>
</select>
</div>
</div>
{interval === "custom" ?
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Value:</label>
<div className="col-sm-9 ">
<input type="number" className="form-control custom-interval" value={customInterval} onChange={this.handleChangeCustomInterval} /><span> meter</span>
</div>
</div>
: ""}
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Layer:</label>
<div className="col-sm-9 ">
<select className="form-control" value={layer} onChange={this.handleSelectLayer}>
{layers.map(l => <option value={l}>{l}</option>)}
</select>
</div>
</div>
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Epsg:</label>
<div className="col-sm-9 ">
<select className="form-control" value={epsg} onChange={this.handleSelectEpsg}>
<option value="4326">WGS84 (EPSG:4326)</option>
<option value="3857">Web Mercator (EPSG:3857)</option>
<option value="custom">Custom EPSG</option>
</select>
</div>
</div>
{epsg === "custom" ?
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">EPSG:</label>
<div className="col-sm-9 ">
<input type="number" className="form-control custom-interval" value={customEpsg} onChange={this.handleChangeCustomEpsg} />
</div>
</div>
: ""}
<div className="text-right action-buttons">
<button onClick={this.handleShowPreview}
disabled={disabled || previewLoading} type="button" className="btn btn-sm btn-primary btn-preview">
{previewLoading ? <i className="fa fa-spin fa-circle-o-notch"/> : <i className="glyphicon glyphicon-eye-open"/>} Preview
</button>
<div className="btn-group">
<button disabled={disabled} type="button" className="btn btn-sm btn-primary" data-toggle="dropdown">
<i className="glyphicon glyphicon-download"></i> Export
</button>
<button disabled={disabled} type="button" className="btn btn-sm dropdown-toggle btn-primary" data-toggle="dropdown"><span className="caret"></span></button>
<ul className="dropdown-menu pull-right">
<li>
<a href="javascript:void(0);">
<i className="fa fa-globe fa-fw"></i> GeoPackage (.GPKG)
</a>
</li>
<li>
<a href="javascript:void(0);">
<i className="fa fa-file-o fa-fw"></i> AutoCAD (.DXF)
</a>
</li>
<li>
<a href="javascript:void(0);">
<i className="fa fa-file-zip-o fa-fw"></i> ShapeFile (.SHP)
</a>
</li>
</ul>
</div>
</div>
</div>);
}
return (<div className="contours-panel">
<span className="close-button" onClick={this.props.onClose}/>
<div className="title">Contours</div>
<hr/>
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Interval:</label>
<div className="col-sm-9 ">
<select className="form-control" value={interval} onChange={this.handleSelectInterval}>
{intervalValues.map(iv => <option value={iv}>{iv} meter</option>)}
<option value="custom">Custom</option>
</select>
</div>
</div>
{interval === "custom" ?
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Value:</label>
<div className="col-sm-9 ">
<input type="number" className="form-control custom-interval" value={customInterval} onChange={this.handleChangeCustomInterval} /><span> meter</span>
</div>
</div>
: ""}
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Layer:</label>
<div className="col-sm-9 ">
<select className="form-control" value={layer} onChange={this.handleSelectLayer}>
<option value="DSM">DSM</option>
</select>
</div>
</div>
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">Projection:</label>
<div className="col-sm-9 ">
<select className="form-control" value={projection} onChange={this.handleSelectProjection}>
<option value="4326">WGS84 (EPSG:4326)</option>
<option value="3857">Web Mercator (EPSG:3857)</option>
<option value="custom">Custom EPSG</option>
</select>
</div>
</div>
{projection === "custom" ?
<div className="row form-group form-inline">
<label className="col-sm-3 control-label">EPSG:</label>
<div className="col-sm-9 ">
<input type="number" className="form-control custom-interval" value={customProjection} onChange={this.handleChangeCustomProjection} />
</div>
</div>
: ""}
<div className="text-right action-buttons">
<button type="button" className="btn btn-sm btn-primary btn-preview">
<i className="glyphicon glyphicon-eye-open"></i> Preview
</button>
<div className="btn-group">
<button type="button" className="btn btn-sm btn-primary" data-toggle="dropdown">
<i className="glyphicon glyphicon-download"></i> Export
</button>
<button type="button" className="btn btn-sm dropdown-toggle btn-primary" data-toggle="dropdown"><span className="caret"></span></button>
<ul className="dropdown-menu">
<li>
<a href="javascript:void(0);">
<i className="fa fa-map-o fa-fw"></i> Orthophoto (GeoTIFF)
</a>
</li>
<li>
<a href="javascript:void(0);">
<i className="fa fa-map-o fa-fw"></i> Orthophoto (GeoTIFF)
</a>
</li>
</ul>
</div>
</div>
{content}
</div>);
}
}

Wyświetl plik

@ -2,6 +2,7 @@
padding: 6px 10px 6px 6px;
background: #fff;
min-width: 250px;
max-width: 300px;
.close-button{
display: inline-block;
@ -57,9 +58,11 @@
.dropdown-menu{
a{
display: inline;
padding-top: 8px;
padding-bottom: 8px;
width: 100%;
text-align: left;
display: block;
padding-top: 0;
padding-bottom: 0;
}
}

Plik binarny nie jest wyświetlany.

Przed

Szerokość:  |  Wysokość:  |  Rozmiar: 1.0 KiB

Wyświetl plik

@ -28,9 +28,9 @@
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="1.979899"
inkscape:cx="-153.03452"
inkscape:cy="0.31522247"
inkscape:zoom="3.959798"
inkscape:cx="-34.955332"
inkscape:cy="20.815973"
inkscape:document-units="mm"
inkscape:current-layer="g831"
showgrid="false"
@ -72,20 +72,20 @@
<g
id="g831"
transform="translate(0.45113764,-0.11484945)">
<ellipse
style="opacity:1;vector-effect:none;fill:#eeeeee;fill-opacity:1;stroke:#191919;stroke-width:1.06403518;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:normal"
id="ellipse827"
ry="4.5824556"
rx="5.6119952"
cy="290.63126"
cx="5.5822544" />
<ellipse
ry="1.9139358"
rx="2.7945361"
cy="290.63126"
cx="5.5822544"
id="path825"
style="opacity:1;vector-effect:none;fill:none;fill-opacity:1;stroke:#191919;stroke-width:1.33004403;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:normal" />
<ellipse
style="opacity:1;vector-effect:none;fill:none;fill-opacity:1;stroke:#191919;stroke-width:1.06403518;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:normal"
id="ellipse827"
cx="5.5822544"
cy="290.63126"
rx="5.6119952"
ry="4.5824556" />
style="opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;stroke:#191919;stroke-width:1.33004403;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:normal" />
</g>
</g>
</g>

Przed

Szerokość:  |  Wysokość:  |  Rozmiar: 3.3 KiB

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 3.3 KiB

Plik binarny nie jest wyświetlany.

Przed

Szerokość:  |  Wysokość:  |  Rozmiar: 2.1 KiB

Wyświetl plik

@ -2,5 +2,13 @@ PluginsAPI.Map.willAddControls([
'contours/build/Contours.js',
'contours/build/Contours.css'
], function(args, Contours){
args.map.addControl(new Contours());
var tasks = [];
for (var i = 0; i < args.tiles.length; i++){
tasks.push(args.tiles[i].meta.task);
}
// TODO: add support for map view where multiple tasks are available?
if (tasks.length === 1){
args.map.addControl(new Contours({tasks: tasks}));
}
});

Wyświetl plik

@ -36,12 +36,15 @@ class TaskVolume(TaskView):
context.add_param('dsm_file', dsm)
context.set_location(dsm)
output = execute_grass_script.delay(os.path.join(
result = execute_grass_script.delay(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"calc_volume.grass"
), context.serialize()).get()
if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error'])
if not isinstance(result, dict): raise GrassEngineException("Unexpected output from GRASS (expected dict)")
if 'error' in result: raise GrassEngineException(result['error'])
output = result.get('output', '')
cols = output.split(':')
if len(cols) == 7:
return Response({'volume': str(abs(float(cols[6])))}, status=status.HTTP_200_OK)

Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "Volume/Area/Length Measurements",
"webodmMinVersion": "0.5.0",
"webodmMinVersion": "0.9.0",
"description": "Compute volume, area and length measurements on Leaflet",
"version": "1.0.0",
"author": "Abdelkoddouss Izem, Piero Toffanin",

Wyświetl plik

@ -86,6 +86,6 @@ def process_pending_tasks():
def execute_grass_script(script, serialized_context = {}):
try:
ctx = grass.create_context(serialized_context)
return ctx.execute(script)
return {'output': ctx.execute(script), 'context': ctx.serialize()}
except GrassEngineException as e:
return {'error': str(e)}