diff --git a/config.js b/config.js index 1248c1e..d0e28e5 100644 --- a/config.js +++ b/config.js @@ -39,6 +39,10 @@ Options: --powercycle When set, the application exits immediately after powering up. Useful for testing launch and compilation issues. --token Sets a token that needs to be passed for every request. This can be used to limit access to the node only to token holders. (default: none) --max_images Specify the maximum number of images that this processing node supports. (default: unlimited) + --callback Specify a callback URL to be invoked when a task completes processing (default: none) + --s3_endpoint Specify a S3 endpoint (for example, nyc3.digitaloceanspaces.com) to upload completed task results to. (default: do not upload to S3) + --s3_access_key S3 access key, required if --s3_endpoint is set. (default: none) + --s3_secret_key S3 secret key, required if --s3_endpoint is set. (default: none) Log Levels: error | debug | info | verbose | debug | silly `); @@ -88,6 +92,9 @@ config.testSkipDems = argv.test_skip_dems || fromConfigFile("testSkipDems", fals config.powercycle = argv.powercycle || fromConfigFile("powercycle", false); config.token = argv.token || fromConfigFile("token", ""); config.maxImages = argv.max_images || fromConfigFile("maxImages", ""); - +config.callback = argv.callback || fromConfigFile("callback", ""); +config.s3Endpoint = argv.s3_endpoint || fromConfigFile("s3Endpoint", "") +config.s3AccessKey = argv.s3_access_key || fromConfigFile("s3AccessKey", process.env.AWS_ACCESS_KEY_ID || "") +config.s3SecretKey = argv.s3_secret_key || fromConfigFile("s3SecretKey", process.env.AWS_SECRET_ACCESS_KEY || "") module.exports = config; diff --git a/libs/S3.js b/libs/S3.js new file mode 100644 index 0000000..2826707 --- /dev/null +++ b/libs/S3.js @@ -0,0 +1,45 @@ +/* +Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap. +Copyright (C) 2016 Node-OpenDroneMap Contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +"use strict"; +const async = require('async'); +const AWS = require('aws-sdk'); +const fs = require('fs'); +const s3 = new AWS.S3({}); + +module.exports = { + uploadToS3: function(srcFolder, endpoint, credentials, cb){ + const PARALLEL_UPLOADS = 5; + + const q = async.queue((task, callback) => { + s3.upload({ + Bucket: 'xxx', + Key: task.dest, + Body: fs.createReadStream(task.src) + }, callback); + }, PARALLEL_UPLOADS); + + q.drain = function() { + console.log('all items have been processed'); + }; + + q.push([ + { src: 'image1.png', dest: 'images/image1.png' }, + { src: 'image2.png', dest: 'images/image2.png' }, + ]); + } +}; diff --git a/package.json b/package.json index 1b7a9cb..b0d79a9 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dependencies": { "archiver": "^1.0.0", "async": "^2.0.0-rc.6", + "aws-sdk": "^2.360.0", "body-parser": "^1.18.3", "express": "^4.16.3", "glob": "^7.1.1",