kopia lustrzana https://github.com/manuelkasper/sotlas-api
Retry object storage uploads several times
rodzic
e3587c2394
commit
8e5e69fcb2
|
@ -32,6 +32,7 @@
|
||||||
"mongodb": "^4.8.1",
|
"mongodb": "^4.8.1",
|
||||||
"multer": "^1.4.2",
|
"multer": "^1.4.2",
|
||||||
"node-cron": "^3.0.1",
|
"node-cron": "^3.0.1",
|
||||||
|
"promise-retry": "^2.0.1",
|
||||||
"reconnect-net": "^1.1.1",
|
"reconnect-net": "^1.1.1",
|
||||||
"sharp": "^0.30.7",
|
"sharp": "^0.30.7",
|
||||||
"togeojson": "^0.16.0",
|
"togeojson": "^0.16.0",
|
||||||
|
@ -1261,6 +1262,11 @@
|
||||||
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
|
||||||
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
|
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
|
||||||
},
|
},
|
||||||
|
"node_modules/err-code": {
|
||||||
|
"version": "2.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
|
||||||
|
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
|
||||||
|
},
|
||||||
"node_modules/es-abstract": {
|
"node_modules/es-abstract": {
|
||||||
"version": "1.20.1",
|
"version": "1.20.1",
|
||||||
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz",
|
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz",
|
||||||
|
@ -3014,6 +3020,18 @@
|
||||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||||
},
|
},
|
||||||
|
"node_modules/promise-retry": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
|
||||||
|
"dependencies": {
|
||||||
|
"err-code": "^2.0.2",
|
||||||
|
"retry": "^0.12.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=10"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/proxy-addr": {
|
"node_modules/proxy-addr": {
|
||||||
"version": "2.0.5",
|
"version": "2.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
|
||||||
|
@ -3252,6 +3270,14 @@
|
||||||
"node": ">=8"
|
"node": ">=8"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/retry": {
|
||||||
|
"version": "0.12.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
|
||||||
|
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 4"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/rimraf": {
|
"node_modules/rimraf": {
|
||||||
"version": "3.0.2",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||||
|
@ -5143,6 +5169,11 @@
|
||||||
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
|
||||||
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
|
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
|
||||||
},
|
},
|
||||||
|
"err-code": {
|
||||||
|
"version": "2.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
|
||||||
|
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
|
||||||
|
},
|
||||||
"es-abstract": {
|
"es-abstract": {
|
||||||
"version": "1.20.1",
|
"version": "1.20.1",
|
||||||
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz",
|
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz",
|
||||||
|
@ -6484,6 +6515,15 @@
|
||||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||||
},
|
},
|
||||||
|
"promise-retry": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
|
||||||
|
"requires": {
|
||||||
|
"err-code": "^2.0.2",
|
||||||
|
"retry": "^0.12.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"proxy-addr": {
|
"proxy-addr": {
|
||||||
"version": "2.0.5",
|
"version": "2.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
|
||||||
|
@ -6679,6 +6719,11 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"retry": {
|
||||||
|
"version": "0.12.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
|
||||||
|
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow=="
|
||||||
|
},
|
||||||
"rimraf": {
|
"rimraf": {
|
||||||
"version": "3.0.2",
|
"version": "3.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||||
|
|
|
@ -32,6 +32,7 @@
|
||||||
"mongodb": "^4.8.1",
|
"mongodb": "^4.8.1",
|
||||||
"multer": "^1.4.2",
|
"multer": "^1.4.2",
|
||||||
"node-cron": "^3.0.1",
|
"node-cron": "^3.0.1",
|
||||||
|
"promise-retry": "^2.0.1",
|
||||||
"reconnect-net": "^1.1.1",
|
"reconnect-net": "^1.1.1",
|
||||||
"sharp": "^0.30.7",
|
"sharp": "^0.30.7",
|
||||||
"togeojson": "^0.16.0",
|
"togeojson": "^0.16.0",
|
||||||
|
|
19
photos.js
19
photos.js
|
@ -6,6 +6,7 @@ const exif = require('exif-reader')
|
||||||
const path = require('path')
|
const path = require('path')
|
||||||
const hasha = require('hasha')
|
const hasha = require('hasha')
|
||||||
const minio = require('minio')
|
const minio = require('minio')
|
||||||
|
const promiseRetry = require('promise-retry')
|
||||||
const config = require('./config')
|
const config = require('./config')
|
||||||
const db = require('./db')
|
const db = require('./db')
|
||||||
|
|
||||||
|
@ -23,7 +24,12 @@ module.exports = {
|
||||||
// Upload original photo to Backblaze (don't wait for completion)
|
// Upload original photo to Backblaze (don't wait for completion)
|
||||||
fsPromises.readFile(filename)
|
fsPromises.readFile(filename)
|
||||||
.then(buffer => {
|
.then(buffer => {
|
||||||
uploadToCloud(config.photos.originalStorage, 'original/' + hashFilename, buffer)
|
promiseRetry((retry, number) => {
|
||||||
|
return uploadToCloud(config.photos.originalStorage, 'original/' + hashFilename, buffer).catch(retry)
|
||||||
|
}, {retries: 5})
|
||||||
|
.catch(() => {
|
||||||
|
console.error(`[ALERT] Cloud photo original upload failed for ${filename}`)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
let photo = {
|
let photo = {
|
||||||
|
@ -87,7 +93,9 @@ module.exports = {
|
||||||
tasks.push(
|
tasks.push(
|
||||||
makeResized(filename, config.photos.sizes[sizeDescr].width, config.photos.sizes[sizeDescr].height)
|
makeResized(filename, config.photos.sizes[sizeDescr].width, config.photos.sizes[sizeDescr].height)
|
||||||
.then(buffer => {
|
.then(buffer => {
|
||||||
return uploadToCloud(config.photos.storage, sizeDescr + '/' + hashFilename, buffer)
|
return promiseRetry((retry, number) => {
|
||||||
|
return uploadToCloud(config.photos.storage, sizeDescr + '/' + hashFilename, buffer).catch(retry)
|
||||||
|
}, {retries: 2})
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
@ -112,13 +120,6 @@ function uploadToCloud(storageConfig, targetPath, buffer) {
|
||||||
'x-amz-acl': 'public-read'
|
'x-amz-acl': 'public-read'
|
||||||
}
|
}
|
||||||
return minioClient.putObject(storageConfig.bucketName, targetPath, buffer, metadata)
|
return minioClient.putObject(storageConfig.bucketName, targetPath, buffer, metadata)
|
||||||
.catch(err => {
|
|
||||||
// Try again
|
|
||||||
return minioClient.putObject(storageConfig.bucketName, targetPath, buffer, metadata)
|
|
||||||
.catch(err => {
|
|
||||||
console.error('[ALERT] Cloud photo upload failed: ' + err)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function getMetadata(src) {
|
function getMetadata(src) {
|
||||||
|
|
Ładowanie…
Reference in New Issue