kopia lustrzana https://github.com/manuelkasper/sotlas-api
Modernize update summits script
rodzic
8d7bbb4986
commit
5db511f6b6
|
@ -12,7 +12,7 @@
|
|||
"@turf/simplify": "^5.1.5",
|
||||
"axios": "^0.21.1",
|
||||
"carrier": "^0.3.0",
|
||||
"csv-parse": "^4.6.3",
|
||||
"csv-parse": "^5.3.0",
|
||||
"diacritics": "^1.3.0",
|
||||
"exif-parser": "^0.1.12",
|
||||
"exif-reader": "^1.0.3",
|
||||
|
@ -33,7 +33,6 @@
|
|||
"multer": "^1.4.2",
|
||||
"nodemailer": "^6.4.6",
|
||||
"reconnect-net": "^1.1.1",
|
||||
"request": "^2.88.0",
|
||||
"sharp": "^0.26.3",
|
||||
"togeojson": "^0.16.0",
|
||||
"togpx": "^0.5.4",
|
||||
|
@ -803,14 +802,6 @@
|
|||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/clone": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
|
||||
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/clone-response": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
|
||||
|
@ -1056,12 +1047,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "4.6.3",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz",
|
||||
"integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==",
|
||||
"dependencies": {
|
||||
"pad": "^3.2.0"
|
||||
}
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz",
|
||||
"integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ=="
|
||||
},
|
||||
"node_modules/dashdash": {
|
||||
"version": "1.14.1",
|
||||
|
@ -1101,14 +1089,6 @@
|
|||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/defaults": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
|
||||
"integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
|
||||
"dependencies": {
|
||||
"clone": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/defer-to-connect": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
|
||||
|
@ -3052,17 +3032,6 @@
|
|||
"node": ">=12.20"
|
||||
}
|
||||
},
|
||||
"node_modules/pad": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
|
||||
"integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
|
||||
"dependencies": {
|
||||
"wcwidth": "^1.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/parse-asn1": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
|
||||
|
@ -4165,14 +4134,6 @@
|
|||
"resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz",
|
||||
"integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y="
|
||||
},
|
||||
"node_modules/wcwidth": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
|
||||
"integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
|
||||
"dependencies": {
|
||||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/web-encoding": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",
|
||||
|
@ -4981,11 +4942,6 @@
|
|||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"clone": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
|
||||
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4="
|
||||
},
|
||||
"clone-response": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz",
|
||||
|
@ -5203,12 +5159,9 @@
|
|||
}
|
||||
},
|
||||
"csv-parse": {
|
||||
"version": "4.6.3",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.6.3.tgz",
|
||||
"integrity": "sha512-pAxEb5kabSaKEwqSXv7vpq6eucXQgY67MLpeLwnYCd21YjTD5OCIIIXGKyUKN/uNQNnzW/elNfxJfozQ1EjB/g==",
|
||||
"requires": {
|
||||
"pad": "^3.2.0"
|
||||
}
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.3.0.tgz",
|
||||
"integrity": "sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ=="
|
||||
},
|
||||
"dashdash": {
|
||||
"version": "1.14.1",
|
||||
|
@ -5239,14 +5192,6 @@
|
|||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
|
||||
},
|
||||
"defaults": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
|
||||
"integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
|
||||
"requires": {
|
||||
"clone": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"defer-to-connect": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
|
||||
|
@ -6765,14 +6710,6 @@
|
|||
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
|
||||
"integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw=="
|
||||
},
|
||||
"pad": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
|
||||
"integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
|
||||
"requires": {
|
||||
"wcwidth": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"parse-asn1": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
|
||||
|
@ -7644,14 +7581,6 @@
|
|||
"resolved": "https://registry.npmjs.org/very-fast-args/-/very-fast-args-1.1.0.tgz",
|
||||
"integrity": "sha1-4W0dH6+KbllqJGQh/ZCneWPQs5Y="
|
||||
},
|
||||
"wcwidth": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
|
||||
"integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
|
||||
"requires": {
|
||||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"web-encoding": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",
|
||||
|
|
|
@ -12,7 +12,7 @@
|
|||
"@turf/simplify": "^5.1.5",
|
||||
"axios": "^0.21.1",
|
||||
"carrier": "^0.3.0",
|
||||
"csv-parse": "^4.6.3",
|
||||
"csv-parse": "^5.3.0",
|
||||
"diacritics": "^1.3.0",
|
||||
"exif-parser": "^0.1.12",
|
||||
"exif-reader": "^1.0.3",
|
||||
|
@ -33,7 +33,6 @@
|
|||
"multer": "^1.4.2",
|
||||
"nodemailer": "^6.4.6",
|
||||
"reconnect-net": "^1.1.1",
|
||||
"request": "^2.88.0",
|
||||
"sharp": "^0.26.3",
|
||||
"togeojson": "^0.16.0",
|
||||
"togpx": "^0.5.4",
|
||||
|
|
|
@ -1,35 +1,28 @@
|
|||
const request = require('request');
|
||||
const axios = require('axios');
|
||||
const MongoClient = require('mongodb').MongoClient;
|
||||
const config = require('../config');
|
||||
const assert = require('assert');
|
||||
const parse = require('csv-parse');
|
||||
const {parse} = require('csv-parse/sync');
|
||||
const fs = require('fs');
|
||||
const removeDiacritics = require('diacritics').remove;
|
||||
|
||||
const client = new MongoClient(config.mongodb.url, {useUnifiedTopology: true});
|
||||
const client = new MongoClient(config.mongodb.url);
|
||||
client.connect(function (err) {
|
||||
assert.equal(null, err);
|
||||
|
||||
processSummitList(client.db(config.mongodb.dbName));
|
||||
});
|
||||
|
||||
function processSummitList(db) {
|
||||
async function processSummitList(db) {
|
||||
let associations = new Map();
|
||||
let now = new Date();
|
||||
|
||||
let prefixToIsoCode = [];
|
||||
parse(fs.readFileSync(__dirname + '/isocodes.txt'), function(err, isocodes) {
|
||||
assert.equal(err, null);
|
||||
prefixToIsoCode = isocodes;
|
||||
});
|
||||
let prefixToIsoCode = parse(fs.readFileSync(__dirname + '/isocodes.txt'));
|
||||
|
||||
request(config.summitListUrl, (error, response, body) => {
|
||||
assert.equal(error, null);
|
||||
let response = await axios.get(config.summitListUrl);
|
||||
let body = response.data.substring(response.data.indexOf("\n")+1, response.data.length);
|
||||
|
||||
body = body.substring(body.indexOf("\n")+1, body.length);
|
||||
|
||||
parse(body, {columns: true, relax_column_count: true}, function(err, summits) {
|
||||
assert.equal(err, null);
|
||||
let summits = parse(body, {columns: true, relax_column_count: true});
|
||||
|
||||
if (summits.length < 100000) {
|
||||
console.error("Bad number of summits, expecting more than 100000");
|
||||
|
@ -72,7 +65,7 @@ function processSummitList(db) {
|
|||
}});
|
||||
|
||||
if (bulkWrites.length >= config.mongodb.batchSize) {
|
||||
db.collection('summits').bulkWrite(bulkWrites);
|
||||
await db.collection('summits').bulkWrite(bulkWrites);
|
||||
bulkWrites = [];
|
||||
}
|
||||
|
||||
|
@ -119,23 +112,22 @@ function processSummitList(db) {
|
|||
}
|
||||
|
||||
if (bulkWrites.length > 0) {
|
||||
db.collection('summits').bulkWrite(bulkWrites);
|
||||
await db.collection('summits').bulkWrite(bulkWrites);
|
||||
}
|
||||
|
||||
// Update associations
|
||||
for (let association of associations.values()) {
|
||||
association.regions = [...association.regions.values()];
|
||||
}
|
||||
|
||||
let session = client.startSession();
|
||||
await session.withTransaction(async () => {
|
||||
let associationCollection = db.collection('associations');
|
||||
associationCollection.deleteMany({}, () => {
|
||||
associationCollection.insertMany([...associations.values()], (err, r) => {
|
||||
if (err)
|
||||
console.error(err);
|
||||
await associationCollection.deleteMany({}, { session });
|
||||
await associationCollection.insertMany([...associations.values()], { session });
|
||||
});
|
||||
session.endSession();
|
||||
client.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function dateToMongo(date, endOfDay = false) {
|
||||
|
|
Ładowanie…
Reference in New Issue