mirror
/
facilmap
kopia lustrzana https://github.com/FacilMap/facilmap
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Respect Nominatim rate limit

pull/172/head
Candid Dauth 2021-05-08 20:49:31 +02:00
rodzic 59f6fd3763
commit 12a25cb486
3 zmienionych plików z 18 dodań i 16 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
server/package.json
Wyświetl plik

@ -62,6 +62,8 @@
"md5-file": "^5.0.0",
"mysql2": "^2.2.5",
"node-cron": "^3.0.0",
"node-fetch": "^2.6.1",
"p-throttle": "^4.1.1",
"promisify-node": "^0.5.0",
"request": "^2.88.2",
"request-debug": "^0.2.0",

27
server/src/search.ts
Wyświetl plik

@ -5,9 +5,10 @@ import zlib from "zlib";
import util from "util";
import { getElevationForPoint, getElevationForPoints } from "./elevation";
import { ZoomLevel, Point, SearchResult } from "facilmap-types";
import request from "./utils/request";
import { Geometry } from "geojson";
import stripBomBuf from "strip-bom-buf";
import fetch from "node-fetch";
import throttle from "p-throttle";
interface NominatimResult {
place_id: number;
@ -75,6 +76,9 @@ const stateAbbr: Record<string, Record<string, string>> = {
}
};
// Respect Nominatim rate limit (https://operations.osmfoundation.org/policies/nominatim/)
const throttledFetch = throttle({ limit: 1, interval: 1000 })(fetch);
interface PointWithZoom extends Point {
zoom?: ZoomLevel;
}
@ -113,10 +117,7 @@ export async function find(query: string, loadUrls = false, loadElevation = fals
}
async function _findQuery(query: string, loadElevation = false): Promise<Array<SearchResult>> {
const body = await request({
url: nameFinderUrl + "/search?format=jsonv2&polygon_geojson=1&addressdetails=1&namedetails=1&limit=" + encodeURIComponent(limit) + "&extratags=1&q=" + encodeURIComponent(query),
json: true
}) as Array<NominatimResult> | NominatimError;
const body: Array<NominatimResult> | NominatimError = await throttledFetch(nameFinderUrl + "/search?format=jsonv2&polygon_geojson=1&addressdetails=1&namedetails=1&limit=" + encodeURIComponent(limit) + "&extratags=1&q=" + encodeURIComponent(query)).then((res) => res.json());
if(!body)
throw new Error("Invalid response from name finder.");
@ -136,10 +137,7 @@ async function _findQuery(query: string, loadElevation = false): Promise<Array<S
}
async function _findOsmObject(type: string, id: string, loadElevation = false): Promise<Array<SearchResult>> {
const body = await request({
url: `${nameFinderUrl}/reverse?format=json&addressdetails=1&polygon_geojson=1&extratags=1&namedetails=1&osm_type=${encodeURI(type.toUpperCase())}&osm_id=${encodeURI(id)}`,
json: true
});
const body = await throttledFetch(`${nameFinderUrl}/reverse?format=json&addressdetails=1&polygon_geojson=1&extratags=1&namedetails=1&osm_type=${encodeURI(type.toUpperCase())}&osm_id=${encodeURI(id)}`).then((res) => res.json());
if(!body || body.error) {
throw new Error(body ? body.error : "Invalid response from name finder");
@ -153,10 +151,7 @@ async function _findOsmObject(type: string, id: string, loadElevation = false):
async function _findLonLat(lonlatWithZoom: PointWithZoom, loadElevation = false): Promise<Array<SearchResult>> {
const [body, elevation] = await Promise.all([
request({
url: `${nameFinderUrl}/reverse?format=json&addressdetails=1&polygon_geojson=0&extratags=1&namedetails=1&lat=${encodeURIComponent(lonlatWithZoom.lat)}&lon=${encodeURIComponent(lonlatWithZoom.lon)}&zoom=${encodeURIComponent(lonlatWithZoom.zoom != null ? (lonlatWithZoom.zoom >= 12 ? lonlatWithZoom.zoom+2 : lonlatWithZoom.zoom) : 17)}`,
json: true
}),
throttledFetch(`${nameFinderUrl}/reverse?format=json&addressdetails=1&polygon_geojson=0&extratags=1&namedetails=1&lat=${encodeURIComponent(lonlatWithZoom.lat)}&lon=${encodeURIComponent(lonlatWithZoom.lon)}&zoom=${encodeURIComponent(lonlatWithZoom.zoom != null ? (lonlatWithZoom.zoom >= 12 ? lonlatWithZoom.zoom+2 : lonlatWithZoom.zoom) : 17)}`).then((res) => res.json()),
...(loadElevation ? [getElevationForPoint(lonlatWithZoom)] : [])
]);
@ -441,7 +436,7 @@ function _formatAddress(result: NominatimResult) {
}
async function _loadUrl(url: string, completeOsmObjects = false) {
let bodyBuf = await request(url, { encoding: null });
let bodyBuf = await fetch(url).then((res) => res.buffer());
if(!bodyBuf)
throw new Error("Invalid response from server.");
@ -483,13 +478,13 @@ async function _loadSubRelations($: cheerio.Root) {
// eslint-disable-next-line no-constant-condition
while (true) {
const promises: Array<ReturnType<typeof request>> = [ ];
const promises: Array<Promise<string>> = [ ];
$("member[type='relation']").each(function(this: cheerio.Element) {
const relId = $(this).attr("ref")!;
if(!loadedIds.has(relId)) {
$(this).remove(); // Remove relation from result, as it will be returned again as part of the sub request
promises.push(request("https://api.openstreetmap.org/api/0.6/relation/" + relId + "/full"));
promises.push(fetch("https://api.openstreetmap.org/api/0.6/relation/" + relId + "/full").then((res) => res.text()));
loadedIds.add(relId);
}
});

5
yarn.lock
Wyświetl plik

@ -6400,6 +6400,11 @@ p-retry@^3.0.1:
dependencies:
retry "^0.12.0"
p-throttle@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/p-throttle/-/p-throttle-4.1.1.tgz#80b1fbd358af40a8bfa1667f9dc8b72b714ad692"
integrity sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g==
p-try@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"