Old task cleanup, zipping

libs/Task.js

@@ -3,6 +3,7 @@ let assert = require('assert');
 let fs = require('fs');
 let rmdir = require('rimraf');
 let odmRunner = require('./odmRunner');
+let AdmZip = require('adm-zip');
 
 let statusCodes = require('./statusCodes');
 
@@ -131,6 +132,25 @@ module.exports = class Task{
 	// Starts processing the task with OpenDroneMap
 	// This will spawn a new process.
 	start(done){
+		function postProcess(done){
+			let zip = new AdmZip();
+			zip.addLocalFolder(`${this.getProjectFolderPath()}/odm_orthophoto`);
+			zip.addLocalFolder(`${this.getProjectFolderPath()}/odm_georeferencing`);
+			zip.addLocalFolder(`${this.getProjectFolderPath()}/odm_texturing`);
+			zip.addLocalFolder(`${this.getProjectFolderPath()}/odm_meshing`);
+			zip.writeZip(`${this.getProjectFolderPath()/all.zip}`);
+			this.setStatus(statusCodes.COMPLETED);
+			finished();
+
+			// TODO: test, think about possibly doing this on the fly
+			// upon download request...
+		}
+
+		function finished(){
+			this.stopTrackingProcessingTime();
+			done();
+		}
+
 		if (this.status.code === statusCodes.QUEUED){
 			this.startTrackingProcessingTime();
 			this.setStatus(statusCodes.RUNNING);
@@ -139,18 +159,20 @@ module.exports = class Task{
 				}, (err, code, signal) => {
 					if (err){
 						this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`});
+						finished();
 					}else{
 						// Don't evaluate if we caused the process to exit via SIGINT?
 						if (this.status.code !== statusCodes.CANCELED){
 							if (code === 0){
-								this.setStatus(statusCodes.COMPLETED);
+								postProcess(done);
 							}else{
 								this.setStatus(statusCodes.FAILED, {errorMessage: `Process exited with code ${code}`});
+								finished();
 							}
+						}else{
+							finished();
 						}
 					}
-					this.stopTrackingProcessingTime();
-					done();
 				}, output => {
 					// Replace console colors
 					output = output.replace(/\x1b\[[0-9;]*m/g, "");

libs/TaskManager.js

@@ -4,9 +4,11 @@ let fs = require('fs');
 let Task = require('./Task');
 let statusCodes = require('./statusCodes');
 let async = require('async');
+let schedule = require('node-schedule');
 
 const PARALLEL_QUEUE_PROCESS_LIMIT = 2;
 const TASKS_DUMP_FILE = "data/tasks.json";
+const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * 3; // 3 days
 
 module.exports = class TaskManager{
 	constructor(done){
@@ -15,12 +17,45 @@ module.exports = class TaskManager{
 
 		async.series([
 			cb => { this.restoreTaskListFromDump(cb); },
+			cb => { this.removeOldTasks(cb); },
 			cb => {
 				this.processNextTask();
+				cb();
+			},
+			cb => {
+				// Every hour
+				schedule.scheduleJob('* 0 * * * *', () => {
+					this.removeOldTasks();
+					this.dumpTaskList();
+				});
+
 				cb();
 			}
-		], done);
-		
+		], done);		
+	}
+
+	// Removes old tasks that have either failed, are completed, or 
+	// have been canceled.
+	removeOldTasks(done){
+		let list = [];
+		let now = new Date().getTime();
+		console.log("Checking for old tasks to be removed...");
+
+		for (let uuid in this.tasks){
+			let task = this.tasks[uuid];
+
+			if ([statusCodes.FAILED, 
+				statusCodes.COMPLETED, 
+				statusCodes.CANCELED].indexOf(task.status.code) !== -1 && 
+				now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN){
+				list.push(task.uuid);
+			}
+		}
+
+		async.eachSeries(list, (uuid, cb) => { 
+			console.log(`Cleaning up old task ${uuid}`)
+			this.remove(uuid, cb); 
+		}, done);
 	}
 
 	// Load tasks that already exists (if any)

package.json

@@ -20,11 +20,13 @@
   },
   "homepage": "https://github.com/pierotofy/node-OpenDroneMap#readme",
   "dependencies": {
+    "adm-zip": "^0.4.7",
     "async": "^2.0.0-rc.6",
     "body-parser": "^1.15.2",
     "express": "^4.14.0",
     "morgan": "^1.7.0",
     "multer": "^1.1.0",
+    "node-schedule": "^1.1.1",
     "node-uuid": "^1.4.7",
     "rimraf": "^2.5.3"
   },