From 42ee410634ee6ff6378778e7b5754d422ed2f91d Mon Sep 17 00:00:00 2001 From: Candid Dauth Date: Thu, 27 Apr 2017 14:02:10 +0200 Subject: [PATCH] Clean up search code --- server/search.js | 689 +++++++++++++++++++++++------------------------ 1 file changed, 344 insertions(+), 345 deletions(-) diff --git a/server/search.js b/server/search.js index 3139c5e0..2d521809 100644 --- a/server/search.js +++ b/server/search.js @@ -1,21 +1,19 @@ -var request = require("request-promise"); -var Promise = require("bluebird"); -var cheerio = require("cheerio"); -var zlib = require("zlib"); -var compressjs = require("compressjs"); - -var utils = require("./utils"); - -request = request.defaults({ +const cheerio = require("cheerio"); +const compressjs = require("compressjs"); +const Promise = require("bluebird"); +const request = require("request-promise").defaults({ gzip: true, headers: { 'User-Agent': process.env.fmUserAgent } }); +const zlib = require("zlib"); -var nameFinderUrl = "https://nominatim.openstreetmap.org"; -var limit = 25; -var stateAbbr = { +const utils = require("./utils"); + +const nameFinderUrl = "https://nominatim.openstreetmap.org"; +const limit = 25; +const stateAbbr = { "us" : { "alabama":"AL","alaska":"AK","arizona":"AZ","arkansas":"AR","california":"CA","colorado":"CO","connecticut":"CT", "delaware":"DE","florida":"FL","georgia":"GA","hawaii":"HI","idaho":"ID","illinois":"IL","indiana":"IN","iowa":"IA", @@ -53,81 +51,44 @@ var stateAbbr = { } }; -function find(query, loadUrls) { - return Promise.resolve().then(function() { - query = query.replace(/^\s+/, "").replace(/\s+$/, ""); - if(loadUrls) { - var m = query.match(/^(node|way|relation)\s+(\d+)$/); - if(m) - return loadUrl("https://api.openstreetmap.org/api/0.6/" + m[1] + "/" + m[2] + (m[1] != "node" ? "/full" : ""), true); +const search = module.exports = { - m = query.match(/^trace\s+(\d+)$/); - if(m) - return loadUrl("https://www.openstreetmap.org/trace/" + m[1] + "/data"); + find(query, loadUrls) { + return Promise.resolve().then(function() { + query = query.replace(/^\s+/, "").replace(/\s+$/, ""); - if(query.match(/^https?:\/\//)) - return loadUrl(query); - } + if(loadUrls) { + let m = query.match(/^(node|way|relation)\s+(\d+)$/); + if(m) + return search._loadUrl("https://api.openstreetmap.org/api/0.6/" + m[1] + "/" + m[2] + (m[1] != "node" ? "/full" : ""), true); - var lonlat_match = query.match(/^(geo\s*:\s*)?(-?\s*\d+([.,]\d+)?)\s*[,;]\s*(-?\s*\d+([.,]\d+)?)(\s*\?z\s*=\s*(\d+))?$/); - var osm_match = query.match(/^([nwr])(\d+)$/i); - if(lonlat_match || osm_match) - { // Reverse search - var url = nameFinderUrl + "/reverse?format=json&addressdetails=1&polygon_geojson=1&extratags=1&namedetails=1"; - var lonlat; + m = query.match(/^trace\s+(\d+)$/); + if(m) + return search._loadUrl("https://www.openstreetmap.org/trace/" + m[1] + "/data"); + if(query.match(/^https?:\/\//)) + return search._loadUrl(query); + } + + let lonlat_match = query.match(/^(geo\s*:\s*)?(-?\s*\d+([.,]\d+)?)\s*[,;]\s*(-?\s*\d+([.,]\d+)?)(\s*\?z\s*=\s*(\d+))?$/); if(lonlat_match) { - lonlat = { + return search._findLonLat({ lat: 1*lonlat_match[2].replace(",", ".").replace(/\s+/, ""), lon : 1*lonlat_match[4].replace(",", ".").replace(/\s+/, ""), zoom : lonlat_match[7] != null ? 1*lonlat_match[7] : null - }; - url += "&lat=" + lonlat.lat - + "&lon=" + lonlat.lon - + "&zoom=" + (lonlat.zoom != null ? (lonlat.zoom >= 12 ? lonlat.zoom+2 : lonlat.zoom) : 17); - } else { - url += "&osm_type=" + osm_match[1].toUpperCase() - + "&osm_id=" + osm_match[2] + }).then((res) => (res.map((res) => (Object.assign(res, {id: query}))))); } - return request({ - url: url, - json: true - }).then(function(body) { - if(!body || body.error) { - if(lonlat) { - var name = utils.round(lonlat.lat, 5) + ", " + utils.round(lonlat.lon, 5); - return [ { - lat: lonlat.lat, - lon : lonlat.lon, - type : "coordinates", - short_name: name, - display_name : name, - zoom: lonlat.zoom != null ? lonlat.zoom : 15, - icon: null - } ]; - } else - throw body ? body.error : "Invalid response from name finder"; - } + let osm_match = query.match(/^([nwr])(\d+)$/i); + if(osm_match) + return search._findOsmObject(osm_match[1], osm_match[2]); - if(lonlat) { - body.lat = lonlat.lat; - body.lon = lonlat.lon; - - if(lonlat.zoom != null) - body.zoom = lonlat.zoom; - } - - var res = prepareSearchResult(body); - - if(lonlat) - res.id = query; - - return [ res ]; - }); - } + return search._findQuery(query); + }); + }, + _findQuery(query) { return request({ url: nameFinderUrl + "/search?format=jsonv2&polygon_geojson=1&addressdetails=1&namedetails=1&limit=" + encodeURIComponent(limit) + "&extratags=1&q=" + encodeURIComponent(query), json: true @@ -138,326 +99,364 @@ function find(query, loadUrls) { if(body.error) throw body.error; - return body.map(prepareSearchResult); + return body.map(search._prepareSearchResult); }); - }); -} + }, -function prepareSearchResult(result) { - var displayName = makeDisplayName(result); - return { - short_name: result.namedetails.name || displayName.split(',')[0], - display_name: displayName, - boundingbox: result.boundingbox, - lat: result.lat, - lon: result.lon, - zoom: result.zoom, - extratags: result.extratags, - geojson: result.geojson, - icon: result.icon && result.icon.replace(/^.*\/([a-z0-9_]+)\.[a-z0-9]+\.[0-9]+\.[a-z0-9]+$/i, "$1"), - type: result.type == "yes" ? result.category : result.type, - id: result.osm_id ? result.osm_type.charAt(0) + result.osm_id : null - }; -} + _findOsmObject(type, id) { + return request({ + url: `${nameFinderUrl}/reverse?format=json&addressdetails=1&polygon_geojson=1&extratags=1&namedetails=1&osm_type=${encodeURI(type.toUpperCase())}&osm_id=${encodeURI(id)}`, + json: true + }).then(function(body) { + if(!body || body.error) { + throw body ? body.error : "Invalid response from name finder"; + } -/** - * Tries to format a search result in a readable way according to the address notation habits in - * the appropriate country. - * @param result {Object} A place object as returned by Nominatim - * @return {String} A readable name for the search result - */ -function makeDisplayName(result) { - // See http://en.wikipedia.org/wiki/Address_%28geography%29#Mailing_address_format_by_country for - // address notation guidelines + return [ search._prepareSearchResult(body) ]; + }); + }, - var type = result.type; - var name = result.namedetails.name; - var countryCode = result.address.country_code; + _findLonLat(lonlatWithZoom) { + return request({ + url: `${nameFinderUrl}/reverse?format=json&addressdetails=1&polygon_geojson=1&extratags=1&namedetails=1&lat=${encodeURI(lonlatWithZoom.lat)}&lon=${encodeURI(lonlatWithZoom.lon)}&zoom=${encodeURI(lonlatWithZoom.zoom != null ? (lonlatWithZoom.zoom >= 12 ? lonlatWithZoom.zoom+2 : lonlatWithZoom.zoom) : 17)}`, + json: true + }).then(function(body) { + if(!body || body.error) { + let name = utils.round(lonlatWithZoom.lat, 5) + ", " + utils.round(lonlatWithZoom.lon, 5); + return [ { + lat: lonlatWithZoom.lat, + lon : lonlatWithZoom.lon, + type : "coordinates", + short_name: name, + display_name : name, + zoom: lonlatWithZoom.zoom != null ? lonlatWithZoom.zoom : 15, + icon: null + } ]; + } - var road = result.address.road; - var housenumber = result.address.house_number; - var suburb = result.address.town || result.address.suburb || result.address.village || result.address.hamlet || result.address.residential; - var postcode = result.address.postcode; - var city = result.address.city; - var county = result.address.county; - var state = result.address.state; - var country = result.address.country; + body.lat = lonlatWithZoom.lat; + body.lon = lonlatWithZoom.lon; - if([ "road", "residential", "town", "suburb", "village", "hamlet", "residential", "city", "county", "state" ].indexOf(type) != -1) - name = ""; + if(lonlatWithZoom.zoom != null) + body.zoom = lonlatWithZoom.zoom; - if(!city && suburb) - { - city = suburb; - suburb = ""; - } + return [ search._prepareSearchResult(body) ]; + }); + }, - if(road) - { - switch(countryCode) + _prepareSearchResult(result) { + let displayName = search._makeDisplayName(result); + return { + short_name: result.namedetails.name || displayName.split(',')[0], + display_name: displayName, + boundingbox: result.boundingbox, + lat: result.lat, + lon: result.lon, + zoom: result.zoom, + extratags: result.extratags, + geojson: result.geojson, + icon: result.icon && result.icon.replace(/^.*\/([a-z0-9_]+)\.[a-z0-9]+\.[0-9]+\.[a-z0-9]+$/i, "$1"), + type: result.type == "yes" ? result.category : result.type, + id: result.osm_id ? result.osm_type.charAt(0) + result.osm_id : null + }; + }, + + /** + * Tries to format a search result in a readable way according to the address notation habits in + * the appropriate country. + * @param result {Object} A place object as returned by Nominatim + * @return {String} A readable name for the search result + */ + _makeDisplayName(result) { + // See http://en.wikipedia.org/wiki/Address_%28geography%29#Mailing_address_format_by_country for + // address notation guidelines + + let type = result.type; + let name = result.namedetails.name; + let countryCode = result.address.country_code; + + let road = result.address.road; + let housenumber = result.address.house_number; + let suburb = result.address.town || result.address.suburb || result.address.village || result.address.hamlet || result.address.residential; + let postcode = result.address.postcode; + let city = result.address.city; + let county = result.address.county; + let state = result.address.state; + let country = result.address.country; + + if([ "road", "residential", "town", "suburb", "village", "hamlet", "residential", "city", "county", "state" ].indexOf(type) != -1) + name = ""; + + if(!city && suburb) { - case "pl": - road = "ul. "+road; - break; - case "ro": - road = "str. "+road; - break; + city = suburb; + suburb = ""; } - } - // Add house number to road - if(road && housenumber) - { + if(road) + { + switch(countryCode) + { + case "pl": + road = "ul. "+road; + break; + case "ro": + road = "str. "+road; + break; + } + } + + // Add house number to road + if(road && housenumber) + { + switch(countryCode) + { + case "ar": + case "at": + case "ca": + case "de": + case "hr": + case "cz": + case "dk": + case "fi": + case "is": + case "il": + case "it": + case "nl": + case "no": + case "pe": + case "pl": + case "sk": + case "si": + case "se": + case "tr": + road += " "+housenumber; + break; + case "be": + case "es": + road += ", "+housenumber; + break; + case "cl": + road += " N° "+housenumber; + break; + case "hu": + road += " "+housenumber+"."; + break; + case "id": + road += " No. "+housenumber; + break; + case "my": + road = "No." +housenumber+", "+road; + break; + case "ro": + road += ", nr. "+road; + break; + case "au": + case "fr": + case "hk": + case "ie": + case "in": + case "nz": + case "sg": + case "lk": + case "tw": + case "gb": + case "us": + default: + road += housenumber+" "+road; + break; + } + } + + // Add postcode and districts to city switch(countryCode) { case "ar": + if(postcode && city) + city = postcode+", "+city; + else if(postcode) + city = postcode; + break; case "at": - case "ca": + case "ch": case "de": + if(city) + { + if(suburb) + city += "-"+(suburb); + suburb = null; + if(type == "suburb" || type == "residential") + type = "city"; + + if(postcode) + city = postcode+" "+city; + } + break; + case "be": case "hr": case "cz": case "dk": case "fi": + case "fr": + case "hu": case "is": case "il": - case "it": + case "my": case "nl": case "no": - case "pe": - case "pl": case "sk": case "si": + case "es": case "se": case "tr": - road += " "+housenumber; - break; - case "be": - case "es": - road += ", "+housenumber; - break; - case "cl": - road += " N° "+housenumber; - break; - case "hu": - road += " "+housenumber+"."; - break; - case "id": - road += " No. "+housenumber; - break; - case "my": - road = "No." +housenumber+", "+road; - break; - case "ro": - road += ", nr. "+road; + if(city && postcode) + city = postcode+" "+city; break; case "au": - case "fr": + case "ca": + case "us": + if(city && state) + { + let thisStateAbbr = stateAbbr[countryCode][state.toLowerCase()]; + if(thisStateAbbr) + { + city += " "+thisStateAbbr; + state = null; + } + } + if(city && postcode) + city += " "+postcode; + else if(postcode) + city = postcode; + break; + case "it": + if(city) + { + if(county) + { + let countyAbbr = stateAbbr.it[county.toLowerCase().replace(/ì/g, "i")]; + if(countyAbbr) + { + city += " ("+countyAbbr+")"; + county = null; + } + } + if(postcode) + city = postcode+" "+city; + } + break; + case "ro": + if(city && county) + { + city += ", jud. "+county; + county = null; + } + if(city && postcode) + city += ", "+postcode; + break; + case "cl": case "hk": + // Postcode rarely/not used case "ie": case "in": + case "id": case "nz": + case "pe": case "sg": case "lk": case "tw": case "gb": - case "us": default: - road += housenumber+" "+road; + if(city && postcode) + city = city+" "+postcode; + else if(postcode) + city = postcode; break; } - } - // Add postcode and districts to city - switch(countryCode) - { - case "ar": - if(postcode && city) - city = postcode+", "+city; - else if(postcode) - city = postcode; - break; - case "at": - case "ch": - case "de": - if(city) - { - if(suburb) - city += "-"+(suburb); - suburb = null; - if(type == "suburb" || type == "residential") - type = "city"; + let ret = [ ]; - if(postcode) - city = postcode+" "+city; - } - break; - case "be": - case "hr": - case "cz": - case "dk": - case "fi": - case "fr": - case "hu": - case "is": - case "il": - case "my": - case "nl": - case "no": - case "sk": - case "si": - case "es": - case "se": - case "tr": - if(city && postcode) - city = postcode+" "+city; - break; - case "au": - case "ca": - case "us": - if(city && state) - { - var thisStateAbbr = stateAbbr[countryCode][state.toLowerCase()]; - if(thisStateAbbr) - { - city += " "+thisStateAbbr; - state = null; - } - } - if(city && postcode) - city += " "+postcode; - else if(postcode) - city = postcode; - break; - case "it": - if(city) - { - if(county) - { - var countyAbbr = stateAbbr.it[county.toLowerCase().replace(/ì/g, "i")]; - if(countyAbbr) - { - city += " ("+countyAbbr+")"; - county = null; - } - } - if(postcode) - city = postcode+" "+city; - } - break; - case "ro": - if(city && county) - { - city += ", jud. "+county; - county = null; - } - if(city && postcode) - city += ", "+postcode; - break; - case "cl": - case "hk": - // Postcode rarely/not used - case "ie": - case "in": - case "id": - case "nz": - case "pe": - case "sg": - case "lk": - case "tw": - case "gb": - default: - if(city && postcode) - city = city+" "+postcode; - else if(postcode) - city = postcode; - break; - } - - var ret = [ ]; - - if(name) - ret.push(name); - if(road) - ret.push(road); - if(suburb) - ret.push(suburb); - if(city) - ret.push(city); - if([ "residential", "town", "suburb", "village", "hamlet", "residential", "city", "county", "state" ].indexOf(type) != -1) - { // Searching for a town - if(county && county != city) - ret.push(county); - if(state && state != city) - ret.push(state); - } - - if(country) - ret.push(country); - - return ret.join(", "); -} - -function loadUrl(url, completeOsmObjects) { - return request(url, { encoding: null }).then(function(bodyBuf) { - if(!bodyBuf) - throw "Invalid response from server."; - - if(bodyBuf[0] == 0x42 && bodyBuf[1] == 0x5a && bodyBuf[2] == 0x68) {// bzip2 - return new Buffer(compressjs.Bzip2.decompressFile(bodyBuf)); + if(name) + ret.push(name); + if(road) + ret.push(road); + if(suburb) + ret.push(suburb); + if(city) + ret.push(city); + if([ "residential", "town", "suburb", "village", "hamlet", "residential", "city", "county", "state" ].indexOf(type) != -1) + { // Searching for a town + if(county && county != city) + ret.push(county); + if(state && state != city) + ret.push(state); } - else if(bodyBuf[0] == 0x1f && bodyBuf[1] == 0x8b && bodyBuf[2] == 0x08) // gzip - return Promise.promisify(zlib.gunzip.bind(zlib))(bodyBuf); - else - return bodyBuf; - }).then(function(bodyBuf) { - var body = bodyBuf.toString(); - if(url.match(/^https?:\/\/www\.freietonne\.de\/seekarte\/getOpenLayerPois\.php\?/)) - return body; - else if(body.match(/^\s* 0) { - return Promise.all(ret).then(function(relations) { - relations.forEach(function(relation) { - $.root().children().append(cheerio.load(relation, { xmlMode: true }).root().children().children()); - }); - - return _loadSubRelations($); + } }); - } else { - return Promise.resolve(); - } -} + }, + + _loadSubRelations($) { + let ret = [ ]; + $("member[type='relation']").each(function() { + let relId = $(this).attr("ref"); + if($("relation[id='" + relId + "']").length == 0) { + ret.push(request("https://api.openstreetmap.org/api/0.6/relation/" + relId + "/full")); + } + }); + + if(ret.length > 0) { + return Promise.all(ret).then(function(relations) { + relations.forEach(function(relation) { + $.root().children().append(cheerio.load(relation, { xmlMode: true }).root().children().children()); + }); + + return search._loadSubRelations($); + }); + } else { + return Promise.resolve(); + } + } -module.exports = { - find: find }; \ No newline at end of file