kopia lustrzana https://github.com/manuelkasper/sotlas-api
62 wiersze
2.2 KiB
JavaScript
62 wiersze
2.2 KiB
JavaScript
const Minio = require('minio')
|
|
const MongoClient = require('mongodb').MongoClient
|
|
const assert = require('assert')
|
|
const config = require('../config')
|
|
|
|
let minAgeForDeletion = 7*86400*1000
|
|
let minExpectedPhotos = 30000
|
|
let maxDeletedPhotos = 1000
|
|
|
|
const client = new MongoClient(config.mongodb.url)
|
|
client.connect(async function (err) {
|
|
assert.equal(null, err)
|
|
let db = client.db(config.mongodb.dbName)
|
|
|
|
// Obtain list of all used filenames from database
|
|
let keepFilenames = new Set(await db.collection('summits').distinct('photos.filename', {'photos.filename': {'$ne': null}}))
|
|
if (keepFilenames.size < minExpectedPhotos) {
|
|
console.error(`Expected at least ${minExpectedPhotos} in DB, found only ${keepFilenames.size}`)
|
|
client.close()
|
|
return
|
|
}
|
|
|
|
// Check for unused files in regular storage
|
|
await deleteFilesNotIn(config.photos.storage, '', keepFilenames)
|
|
|
|
// Check for unused files in original storage
|
|
await deleteFilesNotIn(config.photos.originalStorage, 'original/', keepFilenames)
|
|
|
|
client.close()
|
|
});
|
|
|
|
async function deleteFilesNotIn(storage, prefix, keepFilenames) {
|
|
let allFiles = await listFiles(storage, prefix)
|
|
let filesToDelete = []
|
|
let now = new Date()
|
|
for (let file of allFiles) {
|
|
let baseName = file.name.split(/[\\/]/).pop()
|
|
if (!baseName.endsWith('.jpg') || (now.getTime() - file.lastModified.getTime()) < minAgeForDeletion) {
|
|
continue
|
|
}
|
|
if (!keepFilenames.has(baseName)) {
|
|
filesToDelete.push(file.name)
|
|
}
|
|
}
|
|
if (filesToDelete.length > maxDeletedPhotos) {
|
|
console.error(`Expected at most ${maxDeletedPhotos} files to delete, but found ${filesToDelete.length}`)
|
|
return
|
|
}
|
|
await deleteFiles(storage, filesToDelete)
|
|
console.log(`Deleted ${filesToDelete.length} files from ${storage.endPoint}:${storage.bucketName}`)
|
|
}
|
|
|
|
async function listFiles(storageConfig, targetPath) {
|
|
let minioClient = new Minio.Client(storageConfig)
|
|
return await minioClient.listObjectsV2(storageConfig.bucketName, targetPath, true).toArray()
|
|
}
|
|
|
|
async function deleteFiles(storageConfig, filePaths) {
|
|
let minioClient = new Minio.Client(storageConfig)
|
|
return await minioClient.removeObjects(storageConfig.bucketName, filePaths)
|
|
}
|