From e30718c567ca2f3bd3ddf3f03b32e4dcccccc972 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 23 May 2021 12:33:09 -0400 Subject: [PATCH 01/13] Start on Windows, post-process logic --- helpers/odm_python.bat | 8 ++++++++ libs/Task.js | 32 +++++++++++++++++++++++++++++++- libs/odmInfo.js | 9 +++++++++ libs/odmRunner.js | 22 ++++++++++++++-------- public/index.html | 3 +++ 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 helpers/odm_python.bat diff --git a/helpers/odm_python.bat b/helpers/odm_python.bat new file mode 100644 index 0000000..165d6bf --- /dev/null +++ b/helpers/odm_python.bat @@ -0,0 +1,8 @@ +@echo off + +setlocal + +call %ODM_PATH%\win32env.bat +python %* + +endlocal \ No newline at end of file diff --git a/libs/Task.js b/libs/Task.js index 0429029..c45644c 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -19,12 +19,14 @@ along with this program. If not, see . const config = require('../config'); const async = require('async'); +const os = require('os'); const assert = require('assert'); const logger = require('./logger'); const fs = require('fs'); const path = require('path'); const rmdir = require('rimraf'); const odmRunner = require('./odmRunner'); +const odmInfo = require('./odmInfo'); const processRunner = require('./processRunner'); const Directories = require('./Directories'); const kill = require('tree-kill'); @@ -58,6 +60,26 @@ module.exports = class Task{ this.progress = 0; async.series([ + // Handle post-processing options logic + cb => { + // If we need to post process results + // if pc-ept is supported (build entwine point cloud) + // we automatically add the pc-ept option to the task options by default + if (skipPostProcessing) cb(); + else{ + odmInfo.supportsOption("pc-ept", (err, supported) => { + if (err){ + console.warn(`Cannot check for supported option pc-ept: ${err}`); + }else if (supported){ + if (!this.options.find(opt => opt.name === "pc-ept")){ + this.options.push({ name: 'pc-ept', value: true }); + } + } + cb(); + }); + } + }, + // Read images info cb => { fs.readdir(this.getImagesFolderPath(), (err, files) => { @@ -432,7 +454,15 @@ module.exports = class Task{ } - if (!this.skipPostProcessing) tasks.push(runPostProcessingScript()); + // postprocess.sh is still here for legacy/backward compatibility + // purposes, but we might remove it in the future. The new logic + // instructs the processing engine to do the necessary processing + // of outputs without post processing steps (build EPT). + // We're leaving it here only for Linux/docker setups, but will not + // be triggered on Windows. + if (os.platform() !== "win32" && !this.skipPostProcessing){ + tasks.push(runPostProcessingScript()); + } const taskOutputFile = path.join(this.getProjectFolderPath(), 'task_output.txt'); tasks.push(saveTaskOutput(taskOutputFile)); diff --git a/libs/odmInfo.js b/libs/odmInfo.js index 98b3a94..383e228 100644 --- a/libs/odmInfo.js +++ b/libs/odmInfo.js @@ -58,6 +58,15 @@ module.exports = { }); }, + supportsOption: function(optName, cb){ + this.getOptions((err, json) => { + if (err) cb(err); + else{ + cb(null, !!json.find(opt => opt.name === optName)); + } + }); + }, + getOptions: function(done){ if (odmOptions){ done(null, odmOptions); diff --git a/libs/odmRunner.js b/libs/odmRunner.js index 0cf662a..c49483e 100644 --- a/libs/odmRunner.js +++ b/libs/odmRunner.js @@ -17,6 +17,7 @@ along with this program. If not, see . */ "use strict"; let fs = require('fs'); +let os = require('os'); let path = require('path'); let assert = require('assert'); let spawn = require('child_process').spawn; @@ -28,8 +29,8 @@ module.exports = { run: function(options, projectName, done, outputReceived){ assert(projectName !== undefined, "projectName must be specified"); assert(options["project-path"] !== undefined, "project-path must be defined"); - - const command = path.join(config.odm_path, "run.sh"), + + const command = path.join(config.odm_path, os.platform() === "win32" ? "run.bat" : "run.sh"), params = []; for (var name in options){ @@ -123,6 +124,7 @@ module.exports = { // Launch const env = utils.clone(process.env); env.ODM_OPTIONS_TMP_FILE = utils.tmpPath(".json"); + env.ODM_PATH = config.odm_path; let childProcess = spawn(pythonExe, [path.join(__dirname, "..", "helpers", "odmOptionsToJson.py"), "--project-path", config.odm_path, "bogusname"], { env }); @@ -154,11 +156,15 @@ module.exports = { }) .on('error', handleResult); } - - // Try Python3 first - getOdmOptions("python3", (err, result) => { - if (err) getOdmOptions("python", done); - else done(null, result); - }); + + if (os.platform() === "win32"){ + getOdmOptions("helpers\\odm_python.bat", done); + }else{ + // Try Python3 first + getOdmOptions("python3", (err, result) => { + if (err) getOdmOptions("python", done); + else done(null, result); + }); + } } }; diff --git a/public/index.html b/public/index.html index b190336..d3046a6 100644 --- a/public/index.html +++ b/public/index.html @@ -37,6 +37,9 @@ background-color: #3d74d4; border-color: #4582ec; } + .task{ + background: white; + } From 90ecdb4c3472a43158d04c38b94867e61aadfa36 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 11:19:16 -0400 Subject: [PATCH 02/13] Support for pre-compiled 7z/unzip --- .gitignore | 1 + config.js | 5 +++-- libs/apps.js | 34 ++++++++++++++++++++++++++++++++++ libs/processRunner.js | 7 ++++--- 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 libs/apps.js diff --git a/.gitignore b/.gitignore index 099f639..11fb8de 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ jspm_packages .vscode package-lock.json +apps/ \ No newline at end of file diff --git a/config.js b/config.js index 9c32b26..601b4bb 100644 --- a/config.js +++ b/config.js @@ -20,6 +20,7 @@ along with this program. If not, see . let fs = require('fs'); let argv = require('minimist')(process.argv.slice(2)); let utils = require('./libs/utils'); +let apps = require('./libs/apps'); const spawnSync = require('child_process').spawnSync; if (argv.help){ @@ -141,8 +142,8 @@ config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConc config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1)); // Detect 7z availability -config.has7z = spawnSync("7z", ['--help']).status === 0; -config.hasUnzip = spawnSync("unzip", ['--help']).status === 0; +config.has7z = spawnSync(apps.sevenZ, ['--help']).status === 0; +config.hasUnzip = spawnSync(apps.unzip, ['--help']).status === 0; module.exports = config; diff --git a/libs/apps.js b/libs/apps.js new file mode 100644 index 0000000..76fbbff --- /dev/null +++ b/libs/apps.js @@ -0,0 +1,34 @@ +/* +Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap. +Copyright (C) 2016 Node-OpenDroneMap Contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +const fs = require('fs'); +const path = require('path'); + +let sevenZ = "7z"; +let unzip = "unzip"; + +if (fs.existsSync(path.join("apps", "7z", "7z.exe"))){ + sevenZ = path.resolve(path.join("apps", "7z", "7z.exe")); +} + +if (fs.existsSync(path.join("apps", "unzip", "unzip.exe"))){ + unzip = path.resolve(path.join("apps", "unzip", "unzip.exe")); +} + +module.exports = { + sevenZ, unzip +}; \ No newline at end of file diff --git a/libs/processRunner.js b/libs/processRunner.js index 7c356f2..84eec19 100644 --- a/libs/processRunner.js +++ b/libs/processRunner.js @@ -17,6 +17,7 @@ along with this program. If not, see . */ "use strict"; let fs = require('fs'); +let apps = require('./apps'); let path = require('path'); let assert = require('assert'); let spawn = require('child_process').spawn; @@ -92,14 +93,14 @@ module.exports = { }, ["projectFolderPath"]), - sevenZip: makeRunner("7z", function(options){ + sevenZip: makeRunner(apps.sevenZ, function(options){ return ["a", "-mx=0", "-y", "-r", "-bd", options.destination].concat(options.pathsToArchive); }, ["destination", "pathsToArchive", "cwd"], null, false), - sevenUnzip: makeRunner("7z", function(options){ + sevenUnzip: makeRunner(apps.sevenZ, function(options){ let cmd = "x"; // eXtract files with full paths if (options.noDirectories) cmd = "e"; //Extract files from archive (without using directory names) @@ -109,7 +110,7 @@ module.exports = { null, false), - unzip: makeRunner("unzip", function(options){ + unzip: makeRunner(apps.unzip, function(options){ const opts = options.noDirectories ? ["-j"] : []; return opts.concat(["-qq", "-o", options.file, "-d", options.destination]); }, From 4157ebb38457340825f3de36bbf021b46a616235 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 14:57:51 -0400 Subject: [PATCH 03/13] Winbundle script/command --- .dockerignore | 2 + .gitignore | 4 +- SOURCE | 1 + package.json | 3 +- scripts/winbundle.js | 131 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 SOURCE create mode 100644 scripts/winbundle.js diff --git a/.dockerignore b/.dockerignore index 0cc72dd..c554bbb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ node_modules tests tmp +nodeodm.exe +dist \ No newline at end of file diff --git a/.gitignore b/.gitignore index 11fb8de..ece1052 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ jspm_packages .vscode package-lock.json -apps/ \ No newline at end of file +apps/ +nodeodm.exe +dist/ \ No newline at end of file diff --git a/SOURCE b/SOURCE new file mode 100644 index 0000000..39388f9 --- /dev/null +++ b/SOURCE @@ -0,0 +1 @@ +NodeODM is free software. You can download the source code from https://github.com/OpenDroneMap/NodeODM/ \ No newline at end of file diff --git a/package.json b/package.json index 2cd72b7..55980fe 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "REST API to access ODM", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "winbundle": "node scripts/winbundle.js" }, "repository": { "type": "git", diff --git a/scripts/winbundle.js b/scripts/winbundle.js new file mode 100644 index 0000000..4d43609 --- /dev/null +++ b/scripts/winbundle.js @@ -0,0 +1,131 @@ +const fs = require('fs'); +const spawnSync = require('child_process').spawnSync; +const path = require('path'); +const request = require('request'); +const async = require('async'); +const nodeUnzip = require('node-unzip-2'); +const archiver = require('archiver'); + +const download = function(uri, filename, callback) { + console.log(`Downloading ${uri}`); + request.head(uri, function(err, res, body) { + if (err) callback(err); + else{ + request(uri).pipe(fs.createWriteStream(filename)).on('close', callback); + } + }); +}; + +function downloadApp(destFolder, appUrl, cb){ + if (!fs.existsSync(destFolder)) fs.mkdirSync(destFolder, { recursive: true }); + else { + cb(); + return; + } + + let zipPath = path.join(destFolder, "download.zip"); + let _called = false; + + const done = (err) => { + if (!_called){ // Bug in nodeUnzip, makes this get called twice + if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath); + _called = true; + cb(err); + } + }; + download(appUrl, zipPath, err => { + if (err) done(err); + else{ + // Unzip + console.log(`Extracting ${zipPath}`); + fs.createReadStream(zipPath).pipe(nodeUnzip.Extract({ path: destFolder })) + .on('close', done) + .on('error', done); + } + }); +} + +async.series([ + cb => { + // Cleanup directories + console.log("Cleaning up folders"); + for (let dir of ["data", "tmp"]){ + for (let entry of fs.readdirSync(dir)){ + if (entry !== ".gitignore"){ + console.log(`Removing ${dir}/${entry}`); + fs.rmdirSync(path.join(dir, entry), { recursive: true }); + } + } + } + cb(); + }, + + cb => { + downloadApp(path.join("apps", "7z"), "https://github.com/OpenDroneMap/NodeODM/releases/download/v2.1.0/7z19.zip", cb); + }, + cb => { + downloadApp(path.join("apps", "unzip"), "https://github.com/OpenDroneMap/NodeODM/releases/download/v2.1.0/unzip600.zip", cb); + }, + cb => { + console.log("Building executable"); + const code = spawnSync('nexe.cmd', ['index.js', '-t', 'windows-x64-12.16.3', '-o', 'nodeodm.exe'], { stdio: "pipe"}).status; + + if (code === 0) cb(); + else cb(new Error(`nexe returned non-zero error code: ${code}`)); + }, + cb => { + // Zip + const outFile = path.join("dist", "nodeodm.zip"); + if (!fs.existsSync("dist")) fs.mkdirSync("dist"); + if (fs.existsSync(outFile)) fs.unlinkSync(outFile); + + let output = fs.createWriteStream(outFile); + let archive = archiver.create('zip', { + zlib: { level: 5 } // Sets the compression level (1 = best speed since most assets are already compressed) + }); + + archive.on('finish', () => { + console.log("Done!"); + cb(); + }); + + archive.on('error', err => { + console.error(`Could not archive .zip file: ${err.message}`); + cb(err); + }); + + const files = [ + "apps", + "data", + "helpers", + "public", + "scripts", + "tmp", + "config-default.json", + "LICENSE", + "SOURCE", + "package.json", + "nodeodm.exe" + ]; + + archive.pipe(output); + files.forEach(file => { + console.log(`Adding ${file}`); + let stat = fs.lstatSync(file); + if (stat.isFile()){ + archive.file(file, {name: path.basename(file)}); + }else if (stat.isDirectory()){ + archive.directory(file, path.basename(file)); + }else{ + logger.error(`Could not add ${file}`); + } + }); + + archive.finalize(); + } +], (err) => { + if (err) console.log(`Bundle failed: ${err}`); + else console.log("Bundle ==> dist/nodeodm.zip"); +}); + + From 30e1038339f3182d6d0b184cbf973e6fdef74d07 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 15:21:29 -0400 Subject: [PATCH 04/13] Change bundle name --- scripts/winbundle.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/winbundle.js b/scripts/winbundle.js index 4d43609..dabcf63 100644 --- a/scripts/winbundle.js +++ b/scripts/winbundle.js @@ -6,6 +6,8 @@ const async = require('async'); const nodeUnzip = require('node-unzip-2'); const archiver = require('archiver'); +const bundleName = "nodeodm-windows-x64.zip"; + const download = function(uri, filename, callback) { console.log(`Downloading ${uri}`); request.head(uri, function(err, res, body) { @@ -75,7 +77,7 @@ async.series([ }, cb => { // Zip - const outFile = path.join("dist", "nodeodm.zip"); + const outFile = path.join("dist", bundleName); if (!fs.existsSync("dist")) fs.mkdirSync("dist"); if (fs.existsSync(outFile)) fs.unlinkSync(outFile); @@ -125,7 +127,7 @@ async.series([ } ], (err) => { if (err) console.log(`Bundle failed: ${err}`); - else console.log("Bundle ==> dist/nodeodm.zip"); + else console.log(`Bundle ==> dist/${bundleName}`); }); From 18a714b2def1628bb5eda653afd34e8c82dcef58 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 17:02:49 -0400 Subject: [PATCH 05/13] Improved taskNew creation by async initialization --- libs/Task.js | 57 +++++---- libs/TaskManager.js | 2 +- libs/taskNew.js | 301 +++++++++++++++++++++++--------------------- 3 files changed, 193 insertions(+), 167 deletions(-) diff --git a/libs/Task.js b/libs/Task.js index c45644c..08114a5 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -38,9 +38,8 @@ const archiver = require('archiver'); const statusCodes = require('./statusCodes'); module.exports = class Task{ - constructor(uuid, name, options = [], webhook = null, skipPostProcessing = false, outputs = [], dateCreated = new Date().getTime(), done = () => {}){ + constructor(uuid, name, options = [], webhook = null, skipPostProcessing = false, outputs = [], dateCreated = new Date().getTime(), imagesCountEstimate = -1){ assert(uuid !== undefined, "uuid must be set"); - assert(done !== undefined, "ready must be set"); this.uuid = uuid; this.name = name !== "" ? name : "Task of " + (new Date()).toISOString(); @@ -58,14 +57,18 @@ module.exports = class Task{ this.skipPostProcessing = skipPostProcessing; this.outputs = utils.parseUnsafePathsList(outputs); this.progress = 0; - - async.series([ + this.imagesCountEstimate = imagesCountEstimate; + this.initialized = false; + } + + initialize(done, additionalSteps = []){ + async.series(additionalSteps.concat([ // Handle post-processing options logic cb => { // If we need to post process results // if pc-ept is supported (build entwine point cloud) // we automatically add the pc-ept option to the task options by default - if (skipPostProcessing) cb(); + if (this.skipPostProcessing) cb(); else{ odmInfo.supportsOption("pc-ept", (err, supported) => { if (err){ @@ -113,35 +116,37 @@ module.exports = class Task{ } }); } - ], err => { + ]), err => { + this.initialized = true; done(err, this); }); } static CreateFromSerialized(taskJson, done){ - new Task(taskJson.uuid, + const task = new Task(taskJson.uuid, taskJson.name, - taskJson.options, + taskJson.options, taskJson.webhook, taskJson.skipPostProcessing, taskJson.outputs, - taskJson.dateCreated, - (err, task) => { - if (err) done(err); - else{ - // Override default values with those - // provided in the taskJson - for (let k in taskJson){ - task[k] = taskJson[k]; - } - - // Tasks that were running should be put back to QUEUED state - if (task.status.code === statusCodes.RUNNING){ - task.status.code = statusCodes.QUEUED; - } - done(null, task); + taskJson.dateCreated); + + task.initialize((err, task) => { + if (err) done(err); + else{ + // Override default values with those + // provided in the taskJson + for (let k in taskJson){ + task[k] = taskJson[k]; } - }); + + // Tasks that were running should be put back to QUEUED state + if (task.status.code === statusCodes.RUNNING){ + task.status.code = statusCodes.QUEUED; + } + done(null, task); + } + }); } // Get path where images are stored for this task @@ -578,7 +583,7 @@ module.exports = class Task{ // Re-executes the task (by setting it's state back to QUEUED) // Only tasks that have been canceled, completed or have failed can be restarted. restart(options, cb){ - if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){ + if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1 && this.initialized){ this.setStatus(statusCodes.QUEUED); this.dateCreated = new Date().getTime(); this.dateStarted = 0; @@ -601,7 +606,7 @@ module.exports = class Task{ processingTime: this.processingTime, status: this.status, options: this.options, - imagesCount: this.images.length, + imagesCount: this.images !== undefined ? this.images.length : this.imagesCountEstimate, progress: this.progress }; } diff --git a/libs/TaskManager.js b/libs/TaskManager.js index a0fffcb..c0976df 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -184,7 +184,7 @@ class TaskManager{ // Finds the first QUEUED task. findNextTaskToProcess(){ for (let uuid in this.tasks){ - if (this.tasks[uuid].getStatus() === statusCodes.QUEUED){ + if (this.tasks[uuid].getStatus() === statusCodes.QUEUED && this.tasks[uuid].initialized){ return this.tasks[uuid]; } } diff --git a/libs/taskNew.js b/libs/taskNew.js index 9692d54..11d1d51 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -30,7 +30,7 @@ const async = require('async'); const odmInfo = require('./odmInfo'); const request = require('request'); const ziputils = require('./ziputils'); -const { cancelJob } = require('node-schedule'); +const statusCodes = require('./statusCodes'); const download = function(uri, filename, callback) { request.head(uri, function(err, res, body) { @@ -224,10 +224,6 @@ module.exports = { }, createTask: (req, res) => { - // IMPROVEMENT: consider doing the file moving in the background - // and return a response more quickly instead of a long timeout. - req.setTimeout(1000 * 60 * 20); - const srcPath = path.join("tmp", req.id); // Print error message and cleanup @@ -235,26 +231,149 @@ module.exports = { res.json({error}); removeDirectory(srcPath); }; + + let destPath = path.join(Directories.data, req.id); + let destImagesPath = path.join(destPath, "images"); + let destGcpPath = path.join(destPath, "gcp"); + + const checkMaxImageLimits = (cb) => { + if (!config.maxImages) cb(); + else{ + fs.readdir(destImagesPath, (err, files) => { + if (err) cb(err); + else if (files.length > config.maxImages) cb(new Error(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`)); + else cb(); + }); + } + }; + + let initSteps = [ + // Check if dest directory already exists + cb => { + if (req.files && req.files.length > 0) { + fs.stat(destPath, (err, stat) => { + if (err && err.code === 'ENOENT') cb(); + else{ + // Directory already exists, this could happen + // if a previous attempt at upload failed and the user + // used set-uuid to specify the same UUID over the previous run + // Try to remove it + removeDirectory(destPath, err => { + if (err) cb(new Error(`Directory exists and we couldn't remove it.`)); + else cb(); + }); + } + }); + } else { + cb(); + } + }, + + // Unzips zip URL to tmp// (if any) + cb => { + if (req.body.zipurl) { + let archive = "zipurl.zip"; + + upload.storage.getDestination(req, archive, (err, dstPath) => { + if (err) cb(err); + else{ + let archiveDestPath = path.join(dstPath, archive); + + download(req.body.zipurl, archiveDestPath, cb); + } + }); + } else { + cb(); + } + }, + + // Move all uploads to data//images dir (if any) + cb => fs.mkdir(destPath, undefined, cb), + cb => fs.mkdir(destGcpPath, undefined, cb), + cb => mv(srcPath, destImagesPath, cb), + + // Zip files handling + cb => { + const handleSeed = (cb) => { + const seedFileDst = path.join(destPath, "seed.zip"); + + async.series([ + // Move to project root + cb => mv(path.join(destImagesPath, "seed.zip"), seedFileDst, cb), + + // Extract + cb => { + ziputils.unzip(seedFileDst, destPath, cb); + }, + + // Remove + cb => { + fs.exists(seedFileDst, exists => { + if (exists) fs.unlink(seedFileDst, cb); + else cb(); + }); + } + ], cb); + } + + const handleZipUrl = (cb) => { + // Extract images + ziputils.unzip(path.join(destImagesPath, "zipurl.zip"), + destImagesPath, + cb, true); + } + + // Find and handle zip files and extract + fs.readdir(destImagesPath, (err, entries) => { + if (err) cb(err); + else { + async.eachSeries(entries, (entry, cb) => { + if (entry === "seed.zip"){ + handleSeed(cb); + }else if (entry === "zipurl.zip") { + handleZipUrl(cb); + } else cb(); + }, cb); + } + }); + }, + + // Verify max images limit + cb => { + checkMaxImageLimits(cb); + }, + + cb => { + // Find any *.txt (GCP) file and move it to the data//gcp directory + // also remove any lingering zipurl.zip + fs.readdir(destImagesPath, (err, entries) => { + if (err) cb(err); + else { + async.eachSeries(entries, (entry, cb) => { + if (/\.txt$/gi.test(entry)) { + mv(path.join(destImagesPath, entry), path.join(destGcpPath, entry), cb); + }else if (/\.zip$/gi.test(entry)){ + fs.unlink(path.join(destImagesPath, entry), cb); + } else cb(); + }, cb); + } + }); + } + ]; if (req.error !== undefined){ die(req.error); }else{ - let destPath = path.join(Directories.data, req.id); - let destImagesPath = path.join(destPath, "images"); - let destGcpPath = path.join(destPath, "gcp"); - - const checkMaxImageLimits = (cb) => { - if (!config.maxImages) cb(); - else{ - fs.readdir(destImagesPath, (err, files) => { - if (err) cb(err); - else if (files.length > config.maxImages) cb(new Error(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`)); - else cb(); - }); - } - }; + let imagesCountEstimate = -1; async.series([ + cb => { + // Basic path check + fs.exists(srcPath, exists => { + if (exists) cb(); + else cb(new Error(`Invalid UUID`)); + }); + }, cb => { odmInfo.filterOptions(req.body.options, (err, options) => { if (err) cb(err); @@ -264,134 +383,36 @@ module.exports = { } }); }, - - // Check if dest directory already exists cb => { - if (req.files && req.files.length > 0) { - fs.stat(destPath, (err, stat) => { - if (err && err.code === 'ENOENT') cb(); - else{ - // Directory already exists, this could happen - // if a previous attempt at upload failed and the user - // used set-uuid to specify the same UUID over the previous run - // Try to remove it - removeDirectory(destPath, err => { - if (err) cb(new Error(`Directory exists and we couldn't remove it.`)); - else cb(); - }); - } - }); - } else { + fs.readdir(srcPath, (err, entries) => { + if (!err) imagesCountEstimate = entries.length; cb(); - } - }, - - // Unzips zip URL to tmp// (if any) - cb => { - if (req.body.zipurl) { - let archive = "zipurl.zip"; - - upload.storage.getDestination(req, archive, (err, dstPath) => { - if (err) cb(err); - else{ - let archiveDestPath = path.join(dstPath, archive); - - download(req.body.zipurl, archiveDestPath, cb); - } - }); - } else { - cb(); - } - }, - - // Move all uploads to data//images dir (if any) - cb => fs.mkdir(destPath, undefined, cb), - cb => fs.mkdir(destGcpPath, undefined, cb), - cb => mv(srcPath, destImagesPath, cb), - - // Zip files handling - cb => { - const handleSeed = (cb) => { - const seedFileDst = path.join(destPath, "seed.zip"); - - async.series([ - // Move to project root - cb => mv(path.join(destImagesPath, "seed.zip"), seedFileDst, cb), - - // Extract - cb => { - ziputils.unzip(seedFileDst, destPath, cb); - }, - - // Remove - cb => { - fs.exists(seedFileDst, exists => { - if (exists) fs.unlink(seedFileDst, cb); - else cb(); - }); - } - ], cb); - } - - const handleZipUrl = (cb) => { - // Extract images - ziputils.unzip(path.join(destImagesPath, "zipurl.zip"), - destImagesPath, - cb, true); - } - - // Find and handle zip files and extract - fs.readdir(destImagesPath, (err, entries) => { - if (err) cb(err); - else { - async.eachSeries(entries, (entry, cb) => { - if (entry === "seed.zip"){ - handleSeed(cb); - }else if (entry === "zipurl.zip") { - handleZipUrl(cb); - } else cb(); - }, cb); - } }); }, - - // Verify max images limit cb => { - checkMaxImageLimits(cb); - }, + const task = new Task(req.id, req.body.name, req.body.options, + req.body.webhook, + req.body.skipPostProcessing === 'true', + req.body.outputs, + req.body.dateCreated, + imagesCountEstimate + ); + TaskManager.singleton().addNew(task); + res.json({ uuid: req.id }); + cb(); - cb => { - // Find any *.txt (GCP) file and move it to the data//gcp directory - // also remove any lingering zipurl.zip - fs.readdir(destImagesPath, (err, entries) => { - if (err) cb(err); - else { - async.eachSeries(entries, (entry, cb) => { - if (/\.txt$/gi.test(entry)) { - mv(path.join(destImagesPath, entry), path.join(destGcpPath, entry), cb); - }else if (/\.zip$/gi.test(entry)){ - fs.unlink(path.join(destImagesPath, entry), cb); - } else cb(); - }, cb); - } - }); - }, + // We return a UUID right away but continue + // doing processing in the background - // Create task - cb => { - new Task(req.id, req.body.name, req.body.options, - req.body.webhook, - req.body.skipPostProcessing === 'true', - req.body.outputs, - req.body.dateCreated, - (err, task) => { - if (err) cb(err); - else { - TaskManager.singleton().addNew(task); - res.json({ uuid: req.id }); - cb(); - } - }); + task.initialize(err => { + if (err) { + task.setStatus(statusCodes.FAILED, { errorMessage: err.message }); + + // Cleanup + removeDirectory(srcPath); + removeDirectory(destPath); + } else TaskManager.singleton().processNextTask(); + }, initSteps); } ], err => { if (err) die(err.message); From bb988fb94e29c9eccdb25e528db1859241b5bbea Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 17:54:01 -0400 Subject: [PATCH 06/13] Autobuild bundle for Windows --- .github/workflows/publish-windows.yml | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/publish-windows.yml diff --git a/.github/workflows/publish-windows.yml b/.github/workflows/publish-windows.yml new file mode 100644 index 0000000..8ad290a --- /dev/null +++ b/.github/workflows/publish-windows.yml @@ -0,0 +1,43 @@ +name: Publish Windows Bundle + +on: + push: + branches: + - master + - win32 + tags: + - v* + +jobs: + build: + runs-on: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup NodeJS + uses: actions/setup-node@v2 + with: + node-version: '14' + - name: Setup Env + run: | + npm i + npm i -g nexe + - name: Build bundle + run: | + npm run winbundle + python configure.py dist --signtool-path $((Get-Command signtool).Source) --code-sign-cert-path $env:CODE_SIGN_CERT_PATH + - name: Upload Bundle File + uses: actions/upload-artifact@v2 + with: + name: Bundle + path: dist\*.zip + - name: Upload Bundle to Release + uses: svenstaro/upload-release-action@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: dist\*.zip + file_glob: true + tag: ${{ github.ref }} + overwrite: true + From c87f1f8679559e474e0c09f0d0b9249598d55d5a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 17:58:43 -0400 Subject: [PATCH 07/13] Windows note --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5056f98..10d1042 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,14 @@ You're in good shape! See https://github.com/NVIDIA/nvidia-docker and https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker for information on docker/NVIDIA setup. +### Windows Bundle + +NodeODM can run as a self-contained executable on Windows without the need for additional dependencies (except for [ODM](https://github.com/OpenDroneMap/ODM) which needs to be installed separately). You can download the latest `nodeodm-windows-x64.zip` bundle from the [releases](https://github.com/OpenDroneMap/NodeODM/releases) page. Extract the contents in a folder and run: + +```bash +nodeodm.exe --odm_path c:\path\to\ODM +``` + ### Run it Natively If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu natively you can follow these steps: From 9bdf410e27ba59b17ab69678d892882841e19242 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 24 May 2021 18:00:00 -0400 Subject: [PATCH 08/13] Fix bundle build --- .github/workflows/publish-windows.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish-windows.yml b/.github/workflows/publish-windows.yml index 8ad290a..c1e50e2 100644 --- a/.github/workflows/publish-windows.yml +++ b/.github/workflows/publish-windows.yml @@ -25,7 +25,6 @@ jobs: - name: Build bundle run: | npm run winbundle - python configure.py dist --signtool-path $((Get-Command signtool).Source) --code-sign-cert-path $env:CODE_SIGN_CERT_PATH - name: Upload Bundle File uses: actions/upload-artifact@v2 with: From 7dca5efaeeced2926536c6946b66745125b09a13 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 25 May 2021 10:11:11 -0400 Subject: [PATCH 09/13] Default tasks to running on creation, bump version, fix exit code --- libs/Task.js | 4 +++- libs/odmRunner.js | 4 +++- libs/taskNew.js | 2 -- package.json | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libs/Task.js b/libs/Task.js index 08114a5..d9c6d51 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -46,7 +46,7 @@ module.exports = class Task{ this.dateCreated = isNaN(parseInt(dateCreated)) ? new Date().getTime() : parseInt(dateCreated); this.dateStarted = 0; this.processingTime = -1; - this.setStatus(statusCodes.QUEUED); + this.setStatus(statusCodes.RUNNING); this.options = options; this.gcpFiles = []; this.geoFiles = []; @@ -117,6 +117,8 @@ module.exports = class Task{ }); } ]), err => { + if (err) this.setStatus(statusCodes.FAILED, { errorMessage: err.message }); + else this.setStatus(statusCodes.QUEUED); this.initialized = true; done(err, this); }); diff --git a/libs/odmRunner.js b/libs/odmRunner.js index c49483e..bcf2cb7 100644 --- a/libs/odmRunner.js +++ b/libs/odmRunner.js @@ -71,7 +71,9 @@ module.exports = { } // Launch - let childProcess = spawn(command, params, {cwd: config.odm_path}); + const env = utils.clone(process.env); + env.ODM_NONINTERACTIVE = 1; + let childProcess = spawn(command, params, {cwd: config.odm_path, env}); childProcess .on('exit', (code, signal) => done(null, code, signal)) diff --git a/libs/taskNew.js b/libs/taskNew.js index 11d1d51..1c80ef0 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -406,8 +406,6 @@ module.exports = { task.initialize(err => { if (err) { - task.setStatus(statusCodes.FAILED, { errorMessage: err.message }); - // Cleanup removeDirectory(srcPath); removeDirectory(destPath); diff --git a/package.json b/package.json index 55980fe..a21ad31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "NodeODM", - "version": "2.1.4", + "version": "2.1.5", "description": "REST API to access ODM", "main": "index.js", "scripts": { From fed2142159fcee9262ea728c551adbbd46c38921 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 25 May 2021 10:12:01 -0400 Subject: [PATCH 10/13] Remove branch build --- .github/workflows/publish-windows.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish-windows.yml b/.github/workflows/publish-windows.yml index c1e50e2..696fe20 100644 --- a/.github/workflows/publish-windows.yml +++ b/.github/workflows/publish-windows.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - win32 tags: - v* From baf818e115a8627802928bf5a5dee1a4f89bf44d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 25 May 2021 10:34:32 -0400 Subject: [PATCH 11/13] Fix restart bug, better cleanup handling --- libs/Task.js | 23 +++++++++++++++++++---- libs/taskNew.js | 4 ++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/libs/Task.js b/libs/Task.js index d9c6d51..a6c6db9 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -59,6 +59,7 @@ module.exports = class Task{ this.progress = 0; this.imagesCountEstimate = imagesCountEstimate; this.initialized = false; + this.onInitialize = []; // Events to trigger on initialization } initialize(done, additionalSteps = []){ @@ -117,9 +118,15 @@ module.exports = class Task{ }); } ]), err => { - if (err) this.setStatus(statusCodes.FAILED, { errorMessage: err.message }); - else this.setStatus(statusCodes.QUEUED); + // Status might have changed due to user action + // in which case we leave it unchanged + if (this.getStatus() === statusCodes.RUNNING){ + if (err) this.setStatus(statusCodes.FAILED, { errorMessage: err.message }); + else this.setStatus(statusCodes.QUEUED); + } this.initialized = true; + this.onInitialize.forEach(evt => evt(this)); + this.onInitialize = []; done(err, this); }); } @@ -183,7 +190,10 @@ module.exports = class Task{ // Deletes files and folders related to this task cleanup(cb){ - rmdir(this.getProjectFolderPath(), cb); + if (this.initialized) rmdir(this.getProjectFolderPath(), cb); + else this.onInitialize.push(() => { + rmdir(this.getProjectFolderPath(), cb); + }); } setStatus(code, extra){ @@ -584,8 +594,13 @@ module.exports = class Task{ // Re-executes the task (by setting it's state back to QUEUED) // Only tasks that have been canceled, completed or have failed can be restarted. + // unless they are being initialized, in which case we switch them back to running restart(options, cb){ - if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1 && this.initialized){ + if (!this.initialized && this.status.code === statusCodes.CANCELED){ + this.setStatus(statusCodes.RUNNING); + if (options !== undefined) this.options = options; + cb(null); + }else if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){ this.setStatus(statusCodes.QUEUED); this.dateCreated = new Date().getTime(); this.dateStarted = 0; diff --git a/libs/taskNew.js b/libs/taskNew.js index 1c80ef0..034b66e 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -269,6 +269,10 @@ module.exports = { } }, + cb => { + setTimeout(cb, 5000); + }, + // Unzips zip URL to tmp// (if any) cb => { if (req.body.zipurl) { From 595ee4b11d5e4ca4a7600821e5bceed5d5ef290f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 25 May 2021 10:35:44 -0400 Subject: [PATCH 12/13] Remove debug code --- libs/taskNew.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/taskNew.js b/libs/taskNew.js index 034b66e..1c80ef0 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -269,10 +269,6 @@ module.exports = { } }, - cb => { - setTimeout(cb, 5000); - }, - // Unzips zip URL to tmp// (if any) cb => { if (req.body.zipurl) { From 81896b531627c8465361d326c04e5adfa938609e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 25 May 2021 10:45:30 -0400 Subject: [PATCH 13/13] Remove travis, add github action --- .github/workflows/test-build-prs.yml | 24 ++++++++++++++++++++++++ .travis.yml | 9 --------- 2 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-build-prs.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test-build-prs.yml b/.github/workflows/test-build-prs.yml new file mode 100644 index 0000000..934c2c1 --- /dev/null +++ b/.github/workflows/test-build-prs.yml @@ -0,0 +1,24 @@ +name: Build PRs + +on: + pull_request: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build + uses: docker/build-push-action@v2 + with: + platforms: linux/amd64 + push: false + tags: opendronemap/nodeodm:test + - name: Test Powercycle + run: | + docker run -ti --rm opendronemap/nodeodm:test --powercycle diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 41a53ac..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -sudo: required - -services: - - docker - -before_install: - - docker build -t opendronemap/node-opendronemap . - -script: docker run opendronemap/node-opendronemap --powercycle