Modernize update summits script

master
Manuel Kasper 2022-08-06 10:50:43 +02:00
rodzic 8d7bbb4986
commit 5db511f6b6
3 zmienionych plików z 115 dodań i 195 usunięć

85
package-lock.json wygenerowano
Wyświetl plik

@ -12,7 +12,7 @@
"@turf/simplify": "^5.1.5", "@turf/simplify": "^5.1.5",
"axios": "^0.21.1", "axios": "^0.21.1",
"carrier": "^0.3.0", "carrier": "^0.3.0",
"csv-parse": "^4.6.3", "csv-parse": "^5.3.0",
"diacritics": "^1.3.0", "diacritics": "^1.3.0",
"exif-parser": "^0.1.12", "exif-parser": "^0.1.12",
"exif-reader": "^1.0.3", "exif-reader": "^1.0.3",
@ -33,7 +33,6 @@
"multer": "^1.4.2", "multer": "^1.4.2",
"nodemailer": "^6.4.6", "nodemailer": "^6.4.6",
"reconnect-net": "^1.1.1", "reconnect-net": "^1.1.1",
"request": "^2.88.0",
"sharp": "^0.26.3", "sharp": "^0.26.3",
"togeojson": "^0.16.0", "togeojson": "^0.16.0",
"togpx": "^0.5.4", "togpx": "^0.5.4",
@ -803,14 +802,6 @@
"safe-buffer": "^5.0.1" "safe-buffer": "^5.0.1"
} }
}, },
"node_modules/clone": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
"engines": {
"node": ">=0.8"
}
},
"node_modules/clone-response": { "node_modules/clone-response": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
@ -1056,12 +1047,9 @@
} }
}, },
"node_modules/csv-parse": { "node_modules/csv-parse": {
"version": "4.6.3", "version": "5.3.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz", "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz",
"integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==", "integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ=="
"dependencies": {
"pad": "^3.2.0"
}
}, },
"node_modules/dashdash": { "node_modules/dashdash": {
"version": "1.14.1", "version": "1.14.1",
@ -1101,14 +1089,6 @@
"node": ">=4.0.0" "node": ">=4.0.0"
} }
}, },
"node_modules/defaults": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
"integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
"dependencies": {
"clone": "^1.0.2"
}
},
"node_modules/defer-to-connect": { "node_modules/defer-to-connect": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
@ -3052,17 +3032,6 @@
"node": ">=12.20" "node": ">=12.20"
} }
}, },
"node_modules/pad": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
"integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
"dependencies": {
"wcwidth": "^1.0.1"
},
"engines": {
"node": ">= 4.0.0"
}
},
"node_modules/parse-asn1": { "node_modules/parse-asn1": {
"version": "5.1.6", "version": "5.1.6",
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
@ -4165,14 +4134,6 @@
"resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz", "resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz",
"integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y=" "integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y="
}, },
"node_modules/wcwidth": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
"integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
"dependencies": {
"defaults": "^1.0.3"
}
},
"node_modules/web-encoding": { "node_modules/web-encoding": {
"version": "1.1.5", "version": "1.1.5",
"resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",
@ -4981,11 +4942,6 @@
"safe-buffer": "^5.0.1" "safe-buffer": "^5.0.1"
} }
}, },
"clone": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4="
},
"clone-response": { "clone-response": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
@ -5203,12 +5159,9 @@
} }
}, },
"csv-parse": { "csv-parse": {
"version": "4.6.3", "version": "5.3.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz", "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz",
"integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==", "integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ=="
"requires": {
"pad": "^3.2.0"
}
}, },
"dashdash": { "dashdash": {
"version": "1.14.1", "version": "1.14.1",
@ -5239,14 +5192,6 @@
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
}, },
"defaults": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
"integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
"requires": {
"clone": "^1.0.2"
}
},
"defer-to-connect": { "defer-to-connect": {
"version": "2.0.1", "version": "2.0.1",
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
@ -6765,14 +6710,6 @@
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
"integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==" "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="
}, },
"pad": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
"integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
"requires": {
"wcwidth": "^1.0.1"
}
},
"parse-asn1": { "parse-asn1": {
"version": "5.1.6", "version": "5.1.6",
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
@ -7644,14 +7581,6 @@
"resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz", "resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz",
"integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y=" "integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y="
}, },
"wcwidth": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
"integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
"requires": {
"defaults": "^1.0.3"
}
},
"web-encoding": { "web-encoding": {
"version": "1.1.5", "version": "1.1.5",
"resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",

Wyświetl plik

@ -12,7 +12,7 @@
"@turf/simplify": "^5.1.5", "@turf/simplify": "^5.1.5",
"axios": "^0.21.1", "axios": "^0.21.1",
"carrier": "^0.3.0", "carrier": "^0.3.0",
"csv-parse": "^4.6.3", "csv-parse": "^5.3.0",
"diacritics": "^1.3.0", "diacritics": "^1.3.0",
"exif-parser": "^0.1.12", "exif-parser": "^0.1.12",
"exif-reader": "^1.0.3", "exif-reader": "^1.0.3",
@ -33,7 +33,6 @@
"multer": "^1.4.2", "multer": "^1.4.2",
"nodemailer": "^6.4.6", "nodemailer": "^6.4.6",
"reconnect-net": "^1.1.1", "reconnect-net": "^1.1.1",
"request": "^2.88.0",
"sharp": "^0.26.3", "sharp": "^0.26.3",
"togeojson": "^0.16.0", "togeojson": "^0.16.0",
"togpx": "^0.5.4", "togpx": "^0.5.4",

Wyświetl plik

@ -1,35 +1,28 @@
const request = require('request'); const axios = require('axios');
const MongoClient = require('mongodb').MongoClient; const MongoClient = require('mongodb').MongoClient;
const config = require('../config'); const config = require('../config');
const assert = require('assert'); const assert = require('assert');
const parse = require('csv-parse'); const {parse} = require('csv-parse/sync');
const fs = require('fs'); const fs = require('fs');
const removeDiacritics = require('diacritics').remove; const removeDiacritics = require('diacritics').remove;
const client = new MongoClient(config.mongodb.url, {useUnifiedTopology: true}); const client = new MongoClient(config.mongodb.url);
client.connect(function (err) { client.connect(function (err) {
assert.equal(null, err); assert.equal(null, err);
processSummitList(client.db(config.mongodb.dbName)); processSummitList(client.db(config.mongodb.dbName));
}); });
function processSummitList(db) { async function processSummitList(db) {
let associations = new Map(); let associations = new Map();
let now = new Date(); let now = new Date();
let prefixToIsoCode = []; let prefixToIsoCode = parse(fs.readFileSync(__dirname + '/isocodes.txt'));
parse(fs.readFileSync(__dirname + '/isocodes.txt'), function(err, isocodes) {
assert.equal(err, null);
prefixToIsoCode = isocodes;
});
request(config.summitListUrl, (error, response, body) => { let response = await axios.get(config.summitListUrl);
assert.equal(error, null); let body = response.data.substring(response.data.indexOf("\n")+1, response.data.length);
body = body.substring(body.indexOf("\n")+1, body.length); let summits = parse(body, {columns: true, relax_column_count: true});
parse(body, {columns: true, relax_column_count: true}, function(err, summits) {
assert.equal(err, null);
if (summits.length < 100000) { if (summits.length < 100000) {
console.error("Bad number of summits, expecting more than 100000"); console.error("Bad number of summits, expecting more than 100000");
@ -72,7 +65,7 @@ function processSummitList(db) {
}}); }});
if (bulkWrites.length >= config.mongodb.batchSize) { if (bulkWrites.length >= config.mongodb.batchSize) {
db.collection('summits').bulkWrite(bulkWrites); await db.collection('summits').bulkWrite(bulkWrites);
bulkWrites = []; bulkWrites = [];
} }
@ -119,23 +112,22 @@ function processSummitList(db) {
} }
if (bulkWrites.length > 0) { if (bulkWrites.length > 0) {
db.collection('summits').bulkWrite(bulkWrites); await db.collection('summits').bulkWrite(bulkWrites);
} }
// Update associations
for (let association of associations.values()) { for (let association of associations.values()) {
association.regions = [...association.regions.values()]; association.regions = [...association.regions.values()];
} }
let session = client.startSession();
await session.withTransaction(async () => {
let associationCollection = db.collection('associations'); let associationCollection = db.collection('associations');
associationCollection.deleteMany({}, () => { await associationCollection.deleteMany({}, { session });
associationCollection.insertMany([...associations.values()], (err, r) => { await associationCollection.insertMany([...associations.values()], { session });
if (err) });
console.error(err); session.endSession();
client.close(); client.close();
});
});
});
});
} }
function dateToMongo(date, endOfDay = false) { function dateToMongo(date, endOfDay = false) {