sotlas-api/tools/updateSotaSummits.js

const request = require('request');
const MongoClient = require('mongodb').MongoClient;
const config = require('../config');
const assert = require('assert');
const parse = require('csv-parse');
const fs = require('fs');
const removeDiacritics = require('diacritics').remove;

const client = new MongoClient(config.mongodb.url, {useUnifiedTopology: true});
client.connect(function (err) {
	assert.equal(null, err);
	
	processSummitList(client.db(config.mongodb.dbName));
});

function processSummitList(db) {
	let associations = new Map();
	let now = new Date();

	let prefixToIsoCode = [];
	parse(fs.readFileSync(__dirname + '/isocodes.txt'), function(err, isocodes) {
		assert.equal(err, null);
		prefixToIsoCode = isocodes;
	});
	
	request(config.summitListUrl, (error, response, body) => {
		assert.equal(error, null);
		
		body = body.substring(body.indexOf("\n")+1, body.length);
		
		parse(body, {columns: true, relax_column_count: true}, function(err, summits) {
			assert.equal(err, null);
			
			if (summits.length < 100000) {
				console.error("Bad number of summits, expecting more than 100000");
				client.close();
				return;
			}

			let bulkWrites = [];
			for (let summit of summits) {
				summit.SummitCode = summit.SummitCode.trim();	//anomaly GW/NW-003
				summit.ValidFrom = dateToMongo(summit.ValidFrom);
				summit.ValidTo = dateToMongo(summit.ValidTo, true);
				if (summit.ActivationDate) {
					summit.ActivationDate = dateToMongo(summit.ActivationDate);
				} else {
					summit.ActivationDate = null;
					summit.ActivationCall = null;
				}

				bulkWrites.push({updateOne: {
					filter: {code: summit.SummitCode},
					update: { $set: {
						code: summit.SummitCode,
						name: summit.SummitName,
						nameNd: removeDiacritics(summit.SummitName),
						altitude: parseInt(summit.AltM),
						points: parseInt(summit.Points),
						bonusPoints: parseInt(summit.BonusPoints),
						coordinates: {
							longitude: Number(parseFloat(summit.Longitude).toFixed(5)),
							latitude: Number(parseFloat(summit.Latitude).toFixed(5))
						},
						validFrom: summit.ValidFrom,
						validTo: summit.ValidTo,
						activationCount: parseInt(summit.ActivationCount),
						activationCall: summit.ActivationCall,
						activationDate: summit.ActivationDate
					}},
					upsert: true
				}});

				if (bulkWrites.length >= config.mongodb.batchSize) {
					db.collection('summits').bulkWrite(bulkWrites);
					bulkWrites = [];
				}

				let SummitAssociation = getAssociation(summit.SummitCode);
				let SummitRegion = getRegion(summit.SummitCode);
				
				let isValid = (summit.ValidFrom <= now && summit.ValidTo >= now);
				let association = associations.get(SummitAssociation);
				if (!association) {
					let info = isoCodeForPrefix(SummitAssociation, prefixToIsoCode)
					association = {code: SummitAssociation, name: summit.AssociationName, isoCode: info.isoCode, continent: info.continent, regions: new Map(), summitCount: 0};
					associations.set(SummitAssociation, association);
				}
				let region = association.regions.get(SummitRegion);
				if (!region) {
					region = {code: SummitRegion, name: summit.RegionName, summitCount: 0};
					association.regions.set(SummitRegion, region);
				}
				if (isValid) {
					association.summitCount++;
					region.summitCount++;
				}

				let lat = parseFloat(summit.Latitude);
				let lon = parseFloat(summit.Longitude);

				if (!region.bounds) {
					region.bounds = [[lon, lat], [lon, lat]];
				} else {
					region.bounds[0][0] = Math.min(region.bounds[0][0], lon);
					region.bounds[0][1] = Math.min(region.bounds[0][1], lat);
					region.bounds[1][0] = Math.max(region.bounds[1][0], lon);
					region.bounds[1][1] = Math.max(region.bounds[1][1], lat);
				}

				if (!association.bounds) {
					association.bounds = [[lon, lat], [lon, lat]];
				} else {
					association.bounds[0][0] = Math.min(association.bounds[0][0], lon);
					association.bounds[0][1] = Math.min(association.bounds[0][1], lat);
					association.bounds[1][0] = Math.max(association.bounds[1][0], lon);
					association.bounds[1][1] = Math.max(association.bounds[1][1], lat);
				}
			}

			if (bulkWrites.length > 0) {
				db.collection('summits').bulkWrite(bulkWrites);
			}
			
			for (let association of associations.values()) {
				association.regions = [...association.regions.values()];
			}
			
			let associationCollection = db.collection('associations');
			associationCollection.deleteMany({}, () => {
				associationCollection.insertMany([...associations.values()], (err, r) => {
					if (err)
						console.error(err);
					client.close();
				});
			});
		});
	});
}

function dateToMongo(date, endOfDay = false) {
	let dateRegex = /^(\d\d)\/(\d\d)\/(\d\d\d\d)$/;
	let dateRegex2 = /^(\d\d\d\d)-(\d\d)-(\d\d)/;
	let matches = dateRegex.exec(date);
	let matches2 = dateRegex2.exec(date);
	if (matches) {
		if (endOfDay) {
			return new Date(Date.UTC(matches[3], matches[2]-1, matches[1], 23, 59, 59, 999));
		} else {
			return new Date(Date.UTC(matches[3], matches[2]-1, matches[1]));
		}
	} else if (matches2) {
		if (endOfDay) {
			return new Date(Date.UTC(matches2[1], matches2[2]-1, matches2[3], 23, 59, 59, 999));
		} else {
			return new Date(Date.UTC(matches2[1], matches2[2]-1, matches2[3]));
		}
	} else {
		throw Error("Bad date " + date);
	}
}

let summitRegex = /^(.+)\/(.+)-(\d+)$/;
function getAssociation(summitRef) {
	let matches = summitRegex.exec(summitRef);
	if (matches) {
		return matches[1];
	} else {
		throw Error("Bad summit ref '" + summitRef + "'");
	}
}

function getRegion(summitRef) {
	let matches = summitRegex.exec(summitRef);
	if (matches) {
		return matches[2];
	} else {
		throw Error("Bad summit ref '" + summitRef + "'");
	}
}

function isoCodeForPrefix(prefix, prefixToIsoCode) {
	let isoCodeEnt = prefixToIsoCode.find(el => {
		if (prefix.startsWith(el[0])) {
			return true;
		}
	});
	if (isoCodeEnt) {
		return {isoCode: isoCodeEnt[1], continent: isoCodeEnt[2]};
	} else {
		console.error(`ISO code not found for prefix ${prefix}`);
		return null;
	}
}