Merge pull request #57 from pierotofy/packageUpdate

Package update

Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Piero Toffanin <pt@masseranolabs.com>
 EXPOSE 3000
 
 USER root
-RUN curl --silent --location https://deb.nodesource.com/setup_6.x | bash -
+RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
 RUN apt-get install -y nodejs python-gdal libboost-dev libboost-program-options-dev git cmake
 RUN npm install -g nodemon
 

config.js

@@ -45,6 +45,7 @@ Options:
 	--s3_access_key <key>	S3 access key, required if --s3_endpoint is set. (default: none)
 	--s3_secret_key <secret>	S3 secret key, required if --s3_endpoint is set. (default: none) 
 	--s3_signature_version <version>	S3 signature version. (default: 4)
+	--s3_upload_everything	Upload all task results to S3. (default: upload only .zip archive and orthophoto)
 	--max_concurrency   <number>	Place a cap on the max-concurrency option to use for each task. (default: no limit)
 Log Levels: 
 error | debug | info | verbose | debug | silly 
@@ -101,6 +102,7 @@ config.s3Bucket = argv.s3_bucket || fromConfigFile("s3Bucket", "");
 config.s3AccessKey = argv.s3_access_key || fromConfigFile("s3AccessKey", process.env.AWS_ACCESS_KEY_ID || "")
 config.s3SecretKey = argv.s3_secret_key || fromConfigFile("s3SecretKey", process.env.AWS_SECRET_ACCESS_KEY || "")
 config.s3SignatureVersion = argv.s3_signature_version || fromConfigFile("s3SignatureVersion", "4")
+config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3UploadEverything", false);
 config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0));
 
 module.exports = config;

index.js

@@ -32,7 +32,6 @@ const app = express();
 
 const multer = require('multer');
 const bodyParser = require('body-parser');
-const morgan = require('morgan');
 
 const TaskManager = require('./libs/TaskManager');
 const Task = require('./libs/Task');
@@ -47,7 +46,6 @@ const auth = require('./libs/auth/factory').fromConfig(config);
 const authCheck = auth.getMiddleware();
 const uuidv4 = require('uuid/v4');
 
-
 // zip files
 let request = require('request');
 
@@ -60,20 +58,10 @@ let download = function(uri, filename, callback) {
     });
 };
 
-
-let winstonStream = {
-    write: function(message, encoding) {
-        // Uncomment to get express requests debug output
-        // logger.debug(message.slice(0, -1));
-    }
-};
-app.use(morgan('combined', { stream: winstonStream }));
-app.use(bodyParser.urlencoded({ extended: true }));
-app.use(bodyParser.json());
 app.use(express.static('public'));
 app.use('/swagger.json', express.static('docs/swagger.json'));
 
-let upload = multer({
+const upload = multer({
     storage: multer.diskStorage({
         destination: (req, file, cb) => {
             let dstPath = path.join("tmp", req.id);
@@ -93,6 +81,8 @@ let upload = multer({
     })
 });
 
+const urlEncodedBodyParser = bodyParser.urlencoded({extended: false});
+
 let taskManager;
 let server;
 
@@ -238,7 +228,6 @@ app.post('/task/new', authCheck, (req, res, next) => {
             cb => fs.mkdir(destGpcPath, undefined, cb),
             cb => mv(srcPath, destImagesPath, cb),
 
-
             cb => {
                 // Find any *.zip file and extract
                 fs.readdir(destImagesPath, (err, entries) => {
@@ -465,7 +454,7 @@ app.get('/task/:uuid/download/:asset', authCheck, getTaskFromUuid, (req, res) =>
     if (filePath) {
         if (fs.existsSync(filePath)) {
             res.setHeader('Content-Disposition', `attachment; filename=${asset}`);
-            res.setHeader('Content-Type', mime.lookup(filePath));
+            res.setHeader('Content-Type', mime.getType(filePath));
             res.setHeader('Content-Length', fs.statSync(filePath).size);
 
             const filestream = fs.createReadStream(filePath);
@@ -536,7 +525,7 @@ let successHandler = res => {
  *          schema:
  *            $ref: "#/definitions/Response"
  */
-app.post('/task/cancel', authCheck, uuidCheck, (req, res) => {
+app.post('/task/cancel', urlEncodedBodyParser, authCheck, uuidCheck, (req, res) => {
     taskManager.cancel(req.body.uuid, successHandler(res));
 });
 
@@ -564,7 +553,7 @@ app.post('/task/cancel', authCheck, uuidCheck, (req, res) => {
  *          schema:
  *            $ref: "#/definitions/Response"
  */
-app.post('/task/remove', authCheck, uuidCheck, (req, res) => {
+app.post('/task/remove', urlEncodedBodyParser, authCheck, uuidCheck, (req, res) => {
     taskManager.remove(req.body.uuid, successHandler(res));
 });
 
@@ -599,7 +588,7 @@ app.post('/task/remove', authCheck, uuidCheck, (req, res) => {
  *          schema:
  *            $ref: "#/definitions/Response"
  */
-app.post('/task/restart', authCheck, uuidCheck, (req, res, next) => {
+app.post('/task/restart', urlEncodedBodyParser, authCheck, uuidCheck, (req, res, next) => {
     if (req.body.options){
         odmInfo.filterOptions(req.body.options, (err, options) => {
             if (err) res.json({ error: err.message });

libs/S3.js

@@ -66,7 +66,7 @@ module.exports = {
     uploadPaths: function(srcFolder, bucket, dstFolder, paths, cb, onOutput){
         if (!s3) throw new Error("S3 is not initialized");
 
-        const PARALLEL_UPLOADS = 10;
+        const PARALLEL_UPLOADS = 5;
 
         const q = async.queue((file, done) => {
             logger.debug(`Uploading ${file.src} --> ${file.dest}`);

libs/Task.js

@@ -342,9 +342,14 @@ module.exports = class Task{
             // Upload to S3 all paths + all.zip file (if config says so)
             if (S3.enabled()){
                 tasks.push((done) => {
-                    const s3Paths = !config.test ? 
+                    let s3Paths;
-                                    ['all.zip'].concat(allPaths) : 
+                    if (config.test){
-                                    ['all.zip']; // During testing only upload all.zip
+                        s3Paths = ['all.zip']; // During testing only upload all.zip
+                    }else if (config.s3UploadEverything){
+                        s3Paths = ['all.zip'].concat(allPaths)
+                    }else{
+                        s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif'];
+                    }
 
                     S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths, 
                         err => {

package.json

@@ -20,24 +20,23 @@
   },
   "homepage": "https://github.com/OpenDroneMap/NodeODM#readme",
   "dependencies": {
-    "archiver": "^1.0.0",
+    "archiver": "^3.0.0",
     "async": "^2.6.1",
-    "aws-sdk": "^2.360.0",
+    "aws-sdk": "^2.368.0",
     "body-parser": "^1.18.3",
-    "express": "^4.16.3",
+    "express": "^4.16.4",
-    "glob": "^7.1.1",
+    "glob": "^7.1.3",
-    "mime": "^1.3.4",
+    "mime": "^2.4.0",
     "minimist": "^1.2.0",
-    "morgan": "^1.9.1",
+    "multer": "^1.4.1",
-    "multer": "^1.1.0",
     "mv": "^2.1.1",
-    "node-schedule": "^1.1.1",
+    "node-schedule": "^1.3.1",
     "node-unzip-2": "^0.2.7",
     "request": "^2.88.0",
-    "rimraf": "^2.5.3",
+    "rimraf": "^2.6.2",
-    "swagger-jsdoc": "^1.3.1",
+    "swagger-jsdoc": "^3.2.6",
-    "systeminformation": "^3.42.0",
+    "systeminformation": "^3.51.3",
-    "tree-kill": "^1.1.0",
+    "tree-kill": "^1.2.1",
     "uuid": "^3.3.2",
     "winston": "^2.2.0"
   },