Add: Zip file uploads with optional GCP file
pull/6/head
Lee Pepper 2017-04-07 22:05:20 -06:00
rodzic 9725cba1a2
commit b9ecdde14a
7 zmienionych plików z 775 dodań i 666 usunięć

3
.dockerignore 100644
Wyświetl plik

@ -0,0 +1,3 @@
node_modules
tests
tmp

6
.gitignore vendored
Wyświetl plik

@ -39,3 +39,9 @@ jspm_packages
# Elastic Beanstalk
.elasticbeanstalk
.vscode
.vscode/formatter.json
.vscode/settings.json
.vscode/launch.json

Wyświetl plik

@ -30,8 +30,12 @@ RUN cd /staging/PotreeConverter && \
RUN mkdir /var/www
WORKDIR "/var/www"
RUN git clone https://github.com/OpenDroneMap/node-OpenDroneMap .
#RUN git clone https://github.com/OpenDroneMap/node-OpenDroneMap .
COPY . /var/www
RUN npm install
RUN mkdir tmp
# Fix old version of gdal2tiles.py
# RUN (cd / && patch -p0) <patches/gdal2tiles.patch

Wyświetl plik

@ -38,6 +38,17 @@ let TaskManager = require('./libs/TaskManager');
let Task = require('./libs/Task');
let odmOptions = require('./libs/odmOptions');
let Directories = require('./libs/Directories');
let unzip = require('unzip');
// zip files
let request = require('request');
let download = function(uri, filename, callback) {
request.head(uri, function(err, res, body) {
request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
});
};
let winstonStream = {
write: function(message, encoding) {
@ -50,6 +61,8 @@ app.use(bodyParser.json());
app.use(express.static('public'));
app.use('/swagger.json', express.static('docs/swagger.json'));
let upload = multer({
storage: multer.diskStorage({
destination: (req, file, cb) => {
@ -88,6 +101,12 @@ let server;
* required: true
* type: file
* -
* name: zipurl
* in: formData
* description: Images to process from a url zip file, plus an optional GPC file. If included, the GPC file should have .txt extension
* required: optional
* type: file
* -
* name: name
* in: formData
* description: An optional name to be associated with the task
@ -115,14 +134,19 @@ let server;
* $ref: '#/definitions/Error'
*/
app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
if (!req.files || req.files.length === 0) res.json({error: "Need at least 1 file."});
if ((!req.files || req.files.length === 0) && !req.body.zipurl) res.json({ error: "Need at least 1 file or a zip file url." });
else {
let srcPath = path.join("tmp", req.id);
let destPath = path.join(Directories.data, req.id);
let destImagesPath = path.join(destPath, "images");
let destGpcPath = path.join(destPath, "gpc");
async.series([
// moved these up becaus ei need to populat ethem if zip file first
cb => {
odmOptions.filterOptions(req.body.options, (err, options) => {
if (err) cb(err);
@ -135,14 +159,42 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
// Move all uploads to data/<uuid>/images dir
cb => {
if (req.files && req.files.length > 0) {
setTimeout(function() {
fs.stat(destPath, (err, stat) => {
if (err && err.code === 'ENOENT') cb();
else cb(new Error(`Directory exists (should not have happened: ${err.code})`));
});
}, 500);
} else {
cb();
}
},
cb => fs.mkdir(destPath, undefined, cb),
cb => fs.mkdir(destGpcPath, undefined, cb),
cb => { fs.mkdir(srcPath, undefined, cb) },
cb => { fs.mkdir(destPath, undefined, cb) },
cb => { fs.mkdir(destGpcPath, undefined, cb) },
cb => fs.rename(srcPath, destImagesPath, cb),
cb => {
if (req.body.zipurl) {
var filename = path.basename(req.body.zipurl);
download(req.body.zipurl, destPath + '/' + filename, function() {
// unzip and flatten the zip file (incase there are folders in the zip)
fs.createReadStream(destPath + '/' + filename).pipe(unzip.Parse())
.on('entry', function(entry) {
if (entry.type === 'File') {
entry.pipe(fs.createWriteStream(destImagesPath + '/' + path.basename(entry.path)));
} else {
entry.autodrain();
}
}).on('close', function() {
cb();
});
});
} else {
cb();
}
},
cb => {
// Find any *.txt (GPC) file and move it to the data/<uuid>/gpc directory
fs.readdir(destImagesPath, (err, entries) => {
@ -172,6 +224,7 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
if (err) res.json({ error: err.message });
});
}
});
let getTaskFromUuid = (req, res, next) => {
@ -518,7 +571,8 @@ if (config.test) logger.info("Running in test mode");
let commands = [
cb => odmOptions.initialize(cb),
cb => { taskManager = new TaskManager(cb); },
cb => { server = app.listen(config.port, err => {
cb => {
server = app.listen(config.port, err => {
if (!err) logger.info('Server has started on port ' + String(config.port));
cb(err);
});

Wyświetl plik

@ -31,8 +31,10 @@
"multer": "^1.1.0",
"node-schedule": "^1.1.1",
"node-uuid": "^1.4.7",
"request": "^2.81.0",
"rimraf": "^2.5.3",
"swagger-jsdoc": "^1.3.1",
"unzip": "^0.1.11",
"winston": "^2.2.0"
},
"devDependencies": {

Wyświetl plik

@ -1,5 +1,6 @@
<!doctype html>
<html class="no-js" lang="">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
@ -19,6 +20,7 @@
<script src="js/vendor/modernizr-2.8.3.min.js"></script>
</head>
<body>
<!--[if lt IE 8]>
<p class="browserupgrade">You are using an <strong>outdated</strong> browser. Please <a href="http://browsehappy.com/">upgrade your browser</a> to improve your experience.</p>
@ -27,7 +29,8 @@
<div class="container">
<div class="navbar-header">
<a class="navbar-brand" href="/">Node-OpenDroneMap</a>
<p class="navbar-text navbar-right">Open Source Drone Aerial Imagery Processing</a></p>
<p class="navbar-text navbar-right">Open Source Drone Aerial Imagery Processing</a>
</p>
</div>
</div>
</nav>
@ -42,11 +45,19 @@
<div class="form-group form-inline">
<label for="taskName">Project Name:</lable> <input type="text" class="form-control" value="" id="taskName" />
</div>
<div class="form-group">
<div id="imagesInput" class="form-group">
<label for="images">Aerial Images and GCP List (optional):</label> <input id="images" name="images" multiple type="file">
</div>
<div id="zipFileInput" class="form-group hidden">
<label for="zipurl">Zip file url with Aerial Images and GCP List (optional):</label> <input id="zipurl" name="zipurl" class="form-control" multiple type="text">
<div id="errorBlock" class="help-block"></div>
</div>
<div class="text-right"><input type="submit" class="btn btn-success" value="Start Task" id="btnUpload" /></div>
<div class="text-right"><input type="button" class="btn btn-info" value="From URL" id="btnShowImport" />
<input type="button" class="btn btn-info hidden" value="File Upload" id="btnShowUpload" />
<input type="submit" class="btn btn-success" value="Start Task" id="btnUpload" />
<input type="submit" class="btn btn-success hidden" value="Get ZIP" id="btnImport" /></div>
<div id="options">
<div class="form-inline form-group form-horizontal">
<div data-bind="visible: error(), text: error()" class="alert alert-warning" role="alert"></div>
@ -82,7 +93,9 @@
<div class="task" data-bind="css: {pulsePositive: info().status && info().status.code === 40, pulseNegative: info().status && info().status.code === 30}">
<p data-bind="visible: loading()">Retrieving <span data-bind="text: uuid"></span> ... <span class="glyphicon glyphicon-refresh spinning"></span></p>
<div data-bind="visible: !loading() && !info().error">
<div class="taskItem"><strong>UUID:</strong> <a data-bind="text: info().uuid, attr: {href: '/task/' + info().uuid + '/info'}"></a></div>
<div class="taskItem"><strong>UUID:</strong>
<a data-bind="text: info().uuid, attr: {href: '/task/' + info().uuid + '/info'}"></a>
</div>
<div class="taskItem"><strong>Name:</strong> <span data-bind="text: info().name"></span></div>
<div class="taskItem"><strong>Images:</strong> <span data-bind="text: info().imagesCount"></span></div>
<div class="taskItem"><strong>Status:</strong> <span data-bind="text: statusDescr()"></span></div>
@ -135,12 +148,17 @@
<p>Links: <a href="https://github.com/pierotofy/node-OpenDroneMap/blob/master/docs/index.adoc" target="_blank">API Docs</a> | <a href="https://github.com/OpenDroneMap/WebODM" target="_blank">WebODM</a>
<p>This software is released under the terms of the <a href="https://www.gnu.org/licenses/gpl-3.0.en.html" target="_blank">GPLv3 License</a>. See <a href="https://github.com/pierotofy/node-OpenDroneMap" target="_blank">node-opendronemap</a> on Github for more information.</p>
</footer>
</div> <!-- /container --> <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
<script>window.jQuery || document.write('<script src="js/vendor/jquery-1.11.2.min.js"><\/script>')</script>
</div>
<!-- /container -->
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
<script>
window.jQuery || document.write('<script src="js/vendor/jquery-1.11.2.min.js"><\/script>')
</script>
<script src="js/vendor/bootstrap.min.js"></script>
<script src="js/vendor/knockout-3.4.0.js"></script>
<script src="js/fileinput.js" type="text/javascript"></script>
<script src="js/main.js"></script>
</body>
</html>

Wyświetl plik

@ -49,8 +49,7 @@ $(function(){
TaskList.prototype.saveTaskListToLocalStorage = function() {
localStorage.setItem("odmTaskList", JSON.stringify($.map(this.tasks(), function(task) {
return task.uuid;
})
));
})));
};
TaskList.prototype.remove = function(task) {
this.tasks.remove(function(t) {
@ -279,6 +278,7 @@ $(function(){
uploadExtraData: function() {
return {
name: $("#taskName").val(),
zipurl: $("#zipurl").val(),
options: JSON.stringify(optionsModel.getUserOptions())
};
}
@ -292,6 +292,28 @@ $(function(){
$("#images").fileinput('upload');
});
// zip file control
$('#btnShowImport').on('click', function(e){
e.preventDefault();
$('#zipFileInput').removeClass('hidden');
$('#btnShowUpload').removeClass('hidden');
$('#imagesInput').addClass('hidden');
$('#btnShowImport').addClass('hidden');
});
$('#btnShowUpload').on('click', function(e){
e.preventDefault();
$('#imagesInput').removeClass('hidden');
$('#btnShowImport').removeClass('hidden');
$('#zipFileInput').addClass('hidden');
$('#btnShowUpload').addClass('hidden');
})
var btnUploadLabel = $("#btnUpload").val();
$("#images")
.on('filebatchuploadsuccess', function(e, params) {