const axios = require('axios'); const MongoClient = require('mongodb').MongoClient; const config = require('../config'); const assert = require('assert'); const {parse} = require('csv-parse/sync'); const fs = require('fs'); const removeDiacritics = require('diacritics').remove; const client = new MongoClient(config.mongodb.url); client.connect(function (err) { assert.equal(null, err); processSummitList(client.db(config.mongodb.dbName)); }); async function processSummitList(db) { let associations = new Map(); let now = new Date(); let prefixToIsoCode = parse(fs.readFileSync(__dirname + '/isocodes.txt')); let response = await axios.get(config.summitListUrl); let body = response.data.substring(response.data.indexOf("\n")+1, response.data.length); let summits = parse(body, {columns: true, relax_column_count: true}); if (summits.length < 100000) { console.error("Bad number of summits, expecting more than 100000"); client.close(); return; } let bulkWrites = []; for (let summit of summits) { summit.SummitCode = summit.SummitCode.trim(); //anomaly GW/NW-003 summit.ValidFrom = dateToMongo(summit.ValidFrom); summit.ValidTo = dateToMongo(summit.ValidTo, true); if (summit.ActivationDate) { summit.ActivationDate = dateToMongo(summit.ActivationDate); } else { summit.ActivationDate = null; summit.ActivationCall = null; } bulkWrites.push({updateOne: { filter: {code: summit.SummitCode}, update: { $set: { code: summit.SummitCode, name: summit.SummitName, nameNd: removeDiacritics(summit.SummitName), altitude: parseInt(summit.AltM), points: parseInt(summit.Points), bonusPoints: parseInt(summit.BonusPoints), coordinates: { longitude: Number(parseFloat(summit.Longitude).toFixed(5)), latitude: Number(parseFloat(summit.Latitude).toFixed(5)) }, validFrom: summit.ValidFrom, validTo: summit.ValidTo, activationCount: parseInt(summit.ActivationCount), activationCall: summit.ActivationCall, activationDate: summit.ActivationDate }}, upsert: true }}); if (bulkWrites.length >= config.mongodb.batchSize) { await db.collection('summits').bulkWrite(bulkWrites); bulkWrites = []; } let SummitAssociation = getAssociation(summit.SummitCode); let SummitRegion = getRegion(summit.SummitCode); let isValid = (summit.ValidFrom <= now && summit.ValidTo >= now); let association = associations.get(SummitAssociation); if (!association) { let info = isoCodeForPrefix(SummitAssociation, prefixToIsoCode) association = {code: SummitAssociation, name: summit.AssociationName, isoCode: info.isoCode, continent: info.continent, regions: new Map(), summitCount: 0}; associations.set(SummitAssociation, association); } let region = association.regions.get(SummitRegion); if (!region) { region = {code: SummitRegion, name: summit.RegionName, summitCount: 0}; association.regions.set(SummitRegion, region); } if (isValid) { association.summitCount++; region.summitCount++; } let lat = parseFloat(summit.Latitude); let lon = parseFloat(summit.Longitude); if (!region.bounds) { region.bounds = [[lon, lat], [lon, lat]]; } else { region.bounds[0][0] = Math.min(region.bounds[0][0], lon); region.bounds[0][1] = Math.min(region.bounds[0][1], lat); region.bounds[1][0] = Math.max(region.bounds[1][0], lon); region.bounds[1][1] = Math.max(region.bounds[1][1], lat); } if (!association.bounds) { association.bounds = [[lon, lat], [lon, lat]]; } else { association.bounds[0][0] = Math.min(association.bounds[0][0], lon); association.bounds[0][1] = Math.min(association.bounds[0][1], lat); association.bounds[1][0] = Math.max(association.bounds[1][0], lon); association.bounds[1][1] = Math.max(association.bounds[1][1], lat); } } if (bulkWrites.length > 0) { await db.collection('summits').bulkWrite(bulkWrites); } // Update associations for (let association of associations.values()) { association.regions = [...association.regions.values()]; } let session = client.startSession(); await session.withTransaction(async () => { let associationCollection = db.collection('associations'); await associationCollection.deleteMany({}, { session }); await associationCollection.insertMany([...associations.values()], { session }); }); session.endSession(); client.close(); } function dateToMongo(date, endOfDay = false) { let dateRegex = /^(\d\d)\/(\d\d)\/(\d\d\d\d)$/; let dateRegex2 = /^(\d\d\d\d)-(\d\d)-(\d\d)/; let matches = dateRegex.exec(date); let matches2 = dateRegex2.exec(date); if (matches) { if (endOfDay) { return new Date(Date.UTC(matches[3], matches[2]-1, matches[1], 23, 59, 59, 999)); } else { return new Date(Date.UTC(matches[3], matches[2]-1, matches[1])); } } else if (matches2) { if (endOfDay) { return new Date(Date.UTC(matches2[1], matches2[2]-1, matches2[3], 23, 59, 59, 999)); } else { return new Date(Date.UTC(matches2[1], matches2[2]-1, matches2[3])); } } else { throw Error("Bad date " + date); } } let summitRegex = /^(.+)\/(.+)-(\d+)$/; function getAssociation(summitRef) { let matches = summitRegex.exec(summitRef); if (matches) { return matches[1]; } else { throw Error("Bad summit ref '" + summitRef + "'"); } } function getRegion(summitRef) { let matches = summitRegex.exec(summitRef); if (matches) { return matches[2]; } else { throw Error("Bad summit ref '" + summitRef + "'"); } } function isoCodeForPrefix(prefix, prefixToIsoCode) { let isoCodeEnt = prefixToIsoCode.find(el => { if (prefix.startsWith(el[0])) { return true; } }); if (isoCodeEnt) { return {isoCode: isoCodeEnt[1], continent: isoCodeEnt[2]}; } else { console.error(`ISO code not found for prefix ${prefix}`); return null; } }