From d3e3c513f684e10688c269d15a39d465ea323c63 Mon Sep 17 00:00:00 2001 From: Candid Dauth Date: Sun, 10 Mar 2024 03:18:33 +0100 Subject: [PATCH] Add option to export GPX files as ZIP, add Osmand line width and colour (#246) --- frontend/src/lib/components/export-dialog.vue | 37 ++- .../src/lib/components/ui/export-dropdown.vue | 3 +- frontend/src/lib/components/ui/popover.vue | 10 +- server/package.json | 1 + server/src/database/helpers.ts | 23 +- server/src/database/history.ts | 2 +- server/src/database/line.ts | 19 +- server/src/database/marker.ts | 4 +- server/src/database/route.ts | 39 +-- server/src/database/type.ts | 2 +- server/src/database/view.ts | 2 +- server/src/export/geojson.ts | 77 +++-- server/src/export/gpx-line.ejs | 24 -- server/src/export/gpx.ts | 263 ++++++++++++++---- server/src/frontend.ts | 7 +- server/src/shims.d.ts | 4 +- server/src/socket/socket-v2.ts | 25 +- server/src/utils/__tests__/streams.test.ts | 102 ++++--- server/src/utils/streams.ts | 159 ++++++++--- server/src/utils/utils.ts | 2 +- server/src/webserver.ts | 22 +- utils/src/utils.ts | 20 +- yarn.lock | 140 +++++++++- 23 files changed, 686 insertions(+), 301 deletions(-) delete mode 100644 server/src/export/gpx-line.ejs diff --git a/frontend/src/lib/components/export-dialog.vue b/frontend/src/lib/components/export-dialog.vue index 5cb1644d..cc97001e 100644 --- a/frontend/src/lib/components/export-dialog.vue +++ b/frontend/src/lib/components/export-dialog.vue @@ -44,8 +44,14 @@ ...Object.values(client.value.types).flatMap((type) => type.fields.map((field) => field.name)) ])); + const routeTypeOptions = { + "tracks": "Track points", + "zip": "Track points, one file per line (ZIP file)", + "routes": "Route points" + }; + const format = ref("gpx"); - const useTracks = ref<"1" | "0">("1"); + const routeType = ref("tracks"); const filter = ref(true); const hide = ref(new Set()); const typeId = ref(); @@ -69,7 +75,7 @@ const resolveTypeId = (typeId: ID | undefined) => typeId != null && client.value.types[typeId] ? typeId : undefined; const resolvedTypeId = computed(() => resolveTypeId(typeId.value)); - const canSelectUseTracks = computed(() => format.value === "gpx"); + const canSelectRouteType = computed(() => format.value === "gpx"); const canSelectType = computed(() => format.value === "csv" || (format.value === "table" && method.value === "copy")); const mustSelectType = computed(() => canSelectType.value); const canSelectHide = computed(() => ["table", "csv"].includes(format.value)); @@ -83,8 +89,8 @@ const url = computed(() => { const params = new URLSearchParams(); - if (canSelectUseTracks.value) { - params.set("useTracks", useTracks.value); + if (canSelectRouteType.value) { + params.set("useTracks", routeType.value === "routes" ? "0" : "1"); } if (canSelectHide.value && hide.value.size > 0) { params.set("hide", [...hide.value].join(",")); @@ -130,6 +136,16 @@ ); } + case "gpx": { + return ( + context.baseUrl + + client.value.padData!.id + + `/${format.value}` + + (routeType.value === "zip" ? `/zip` : "") + + (paramsStr ? `?${paramsStr}` : '') + ); + } + default: { return ( context.baseUrl @@ -232,14 +248,18 @@ -
+
diff --git a/frontend/src/lib/components/ui/export-dropdown.vue b/frontend/src/lib/components/ui/export-dropdown.vue index 8a5e7b18..a7ce901f 100644 --- a/frontend/src/lib/components/ui/export-dropdown.vue +++ b/frontend/src/lib/components/ui/export-dropdown.vue @@ -7,6 +7,7 @@ import { useToasts } from "./toasts/toasts.vue"; import { saveAs } from "file-saver"; import vTooltip from "../../utils/tooltip"; + import { getSafeFilename } from "facilmap-utils"; const context = injectContextRequired(); const toasts = useToasts(); @@ -26,7 +27,7 @@ try { const exported = await props.getExport(format); - saveAs(new Blob([exported], { type: "application/gpx+xml" }), `${props.filename}.gpx`); + saveAs(new Blob([exported], { type: "application/gpx+xml" }), `${getSafeFilename(props.filename)}.gpx`); } catch(err: any) { toasts.showErrorToast(`fm${context.id}-export-dropdown-error`, "Error exporting route", err); } finally { diff --git a/frontend/src/lib/components/ui/popover.vue b/frontend/src/lib/components/ui/popover.vue index 4e1d6c13..af6d6154 100644 --- a/frontend/src/lib/components/ui/popover.vue +++ b/frontend/src/lib/components/ui/popover.vue @@ -149,12 +149,20 @@ \ No newline at end of file diff --git a/server/package.json b/server/package.json index 77bdbade..0b6ad354 100644 --- a/server/package.json +++ b/server/package.json @@ -66,6 +66,7 @@ "string-similarity": "^4.0.4", "strip-bom-buf": "^4.0.0", "unzipper": "^0.10.14", + "zip-stream": "^6.0.0", "zod": "^3.22.4" }, "devDependencies": { diff --git a/server/src/database/helpers.ts b/server/src/database/helpers.ts index d286a49f..c2afe378 100644 --- a/server/src/database/helpers.ts +++ b/server/src/database/helpers.ts @@ -118,7 +118,7 @@ export default class DatabaseHelpers { this._db = db; } - async _updateObjectStyles(objects: Marker | Line | AsyncGenerator): Promise { + async _updateObjectStyles(objects: Marker | Line | AsyncIterable): Promise { const types: Record = { }; for await (const object of Symbol.asyncIterator in objects ? objects : arrayToAsyncIterator([objects])) { const padId = object.padId; @@ -187,7 +187,7 @@ export default class DatabaseHelpers { return data; } - async* _getPadObjects(type: string, padId: PadId, condition?: FindOptions): AsyncGenerator { + async* _getPadObjects(type: string, padId: PadId, condition?: FindOptions): AsyncIterable { const includeData = [ "Marker", "Line" ].includes(type); if(includeData) { @@ -390,10 +390,23 @@ export default class DatabaseHelpers { } } - async _bulkCreateInBatches(model: ModelCtor, data: Array>): Promise> { + async _bulkCreateInBatches(model: ModelCtor, data: Iterable> | AsyncIterable>): Promise> { const result: Array = []; - for(let i=0; i it.toJSON())); + let slice: Array> = []; + const createSlice = async () => { + result.push(...(await model.bulkCreate(slice)).map((it) => it.toJSON())); + slice = []; + }; + + for await (const item of data) { + slice.push(item); + if (slice.length >= ITEMS_PER_BATCH) { + createSlice(); + } + } + if (slice.length > 0) { + createSlice(); + } return result; } diff --git a/server/src/database/history.ts b/server/src/database/history.ts index 9ff0bff2..79d12eb5 100644 --- a/server/src/database/history.ts +++ b/server/src/database/history.ts @@ -100,7 +100,7 @@ export default class DatabaseHistory { } - getHistory(padId: PadId, types?: HistoryEntryType[]): AsyncGenerator { + getHistory(padId: PadId, types?: HistoryEntryType[]): AsyncIterable { const query: FindOptions = { order: [[ "time", "DESC" ]] }; if(types) query.where = {type: types}; diff --git a/server/src/database/line.ts b/server/src/database/line.ts index b1d4a392..c7743705 100644 --- a/server/src/database/line.ts +++ b/server/src/database/line.ts @@ -179,22 +179,15 @@ export default class DatabaseLines { this.LineModel.hasMany(this.LineDataModel, { foreignKey: "lineId" }); } - getPadLines(padId: PadId, fields?: Array): AsyncGenerator { + getPadLines(padId: PadId, fields?: Array): AsyncIterable { const cond = fields ? { attributes: fields } : { }; return this._db.helpers._getPadObjects("Line", padId, cond); } - getPadLinesByType(padId: PadId, typeId: ID): AsyncGenerator { + getPadLinesByType(padId: PadId, typeId: ID): AsyncIterable { return this._db.helpers._getPadObjects("Line", padId, { where: { typeId: typeId } }); } - async* getPadLinesWithPoints(padId: PadId): AsyncGenerator { - for await (const line of this.getPadLines(padId)) { - const trackPoints = await this.getAllLinePoints(line.id); - yield { ...line, trackPoints }; - } - } - async getLineTemplate(padId: PadId, data: { typeId: ID }): Promise { const lineTemplate = { ...this.LineModel.build({ ...data, padId: padId } satisfies Partial> as any).toJSON(), @@ -310,7 +303,7 @@ export default class DatabaseLines { return oldLine; } - async* getLinePointsForPad(padId: PadId, bboxWithZoom: BboxWithZoom & BboxWithExcept): AsyncGenerator<{ id: ID; trackPoints: TrackPoint[] }, void, void> { + async* getLinePointsForPad(padId: PadId, bboxWithZoom: BboxWithZoom & BboxWithExcept): AsyncIterable<{ id: ID; trackPoints: TrackPoint[] }> { const lines = await this.LineModel.findAll({ attributes: ["id"], where: { padId } }); const chunks = chunk(lines.map((line) => line.id), 50000); for (const lineIds of chunks) { @@ -336,12 +329,14 @@ export default class DatabaseLines { } } - async getAllLinePoints(lineId: ID): Promise { + async* getAllLinePoints(lineId: ID): AsyncIterable { const points = await this.LineModel.build({ id: lineId } satisfies Partial> as any).getLinePoints({ attributes: [ "pos", "lat", "lon", "ele", "zoom", "idx" ], order: [["idx", "ASC"]] }); - return points.map((point) => omit(point.toJSON(), ["pos"]) as TrackPoint); + for (const point of points) { + yield omit(point.toJSON(), ["pos"]) as TrackPoint; + } } } diff --git a/server/src/database/marker.ts b/server/src/database/marker.ts index be38bd89..e710f0cf 100644 --- a/server/src/database/marker.ts +++ b/server/src/database/marker.ts @@ -76,11 +76,11 @@ export default class DatabaseMarkers { this.MarkerModel.hasMany(this.MarkerDataModel, { foreignKey: "markerId" }); } - getPadMarkers(padId: PadId, bbox?: BboxWithZoom & BboxWithExcept): AsyncGenerator { + getPadMarkers(padId: PadId, bbox?: BboxWithZoom & BboxWithExcept): AsyncIterable { return this._db.helpers._getPadObjects("Marker", padId, { where: this._db.helpers.makeBboxCondition(bbox) }); } - getPadMarkersByType(padId: PadId, typeId: ID): AsyncGenerator { + getPadMarkersByType(padId: PadId, typeId: ID): AsyncIterable { return this._db.helpers._getPadObjects("Marker", padId, { where: { padId: padId, typeId: typeId } }); } diff --git a/server/src/database/route.ts b/server/src/database/route.ts index 1e79db2d..53faab81 100644 --- a/server/src/database/route.ts +++ b/server/src/database/route.ts @@ -6,6 +6,7 @@ import { type BboxWithExcept, createModel, getPosType, getVirtualLatType, getVir import { calculateRouteForLine } from "../routing/routing.js"; import { omit } from "lodash-es"; import type { Point as GeoJsonPoint } from "geojson"; +import { asyncIteratorToArray } from "../utils/streams.js"; const updateTimes: Record = {}; @@ -155,24 +156,24 @@ export default class DatabaseRoutes { return; const line = await this._db.lines.getLine(padId, lineId); - const linePoints = await this._db.lines.getAllLinePoints(lineId); + const linePointsIt = this._db.lines.getAllLinePoints(lineId); + const linePoints = await asyncIteratorToArray((async function*() { + for await (const linePoint of linePointsIt) { + yield { + routeId, + lat: linePoint.lat, + lon: linePoint.lon, + ele: linePoint.ele, + zoom: linePoint.zoom, + idx: linePoint.idx + }; + } + })()); if(thisTime != updateTimes[routeId]) return; - const create = []; - for(const linePoint of linePoints) { - create.push({ - routeId, - lat: linePoint.lat, - lon: linePoint.lon, - ele: linePoint.ele, - zoom: linePoint.zoom, - idx: linePoint.idx - }); - } - - await this._db.helpers._bulkCreateInBatches(this.RoutePointModel, create); + await this._db.helpers._bulkCreateInBatches(this.RoutePointModel, linePoints); if(thisTime != updateTimes[routeId]) return; @@ -214,12 +215,14 @@ export default class DatabaseRoutes { return data.map((d) => omit(d.toJSON(), ["pos"]) as TrackPoint); } - async getAllRoutePoints(routeId: string): Promise { - const data = await this.RoutePointModel.findAll({ - where: {routeId}, + async* getAllRoutePoints(routeId: string): AsyncIterable { + const points = await this.RoutePointModel.findAll({ + where: { routeId }, attributes: [ "pos", "lat", "lon", "idx", "ele", "zoom"] }); - return data.map((d) => omit(d.toJSON(), ["pos"]) as TrackPoint); + for (const point of points) { + yield omit(point.toJSON(), ["pos"]) as TrackPoint; + } } } diff --git a/server/src/database/type.ts b/server/src/database/type.ts index c4806659..103d3b24 100644 --- a/server/src/database/type.ts +++ b/server/src/database/type.ts @@ -85,7 +85,7 @@ export default class DatabaseTypes { PadModel.hasMany(this.TypeModel, { foreignKey: "padId" }); } - getTypes(padId: PadId): AsyncGenerator { + getTypes(padId: PadId): AsyncIterable { return this._db.helpers._getPadObjects("Type", padId); } diff --git a/server/src/database/view.ts b/server/src/database/view.ts index 592a2482..b5038237 100644 --- a/server/src/database/view.ts +++ b/server/src/database/view.ts @@ -57,7 +57,7 @@ export default class DatabaseViews { this._db.pads.PadModel.hasMany(this.ViewModel, { foreignKey: "padId" }); } - getViews(padId: PadId): AsyncGenerator { + getViews(padId: PadId): AsyncIterable { return this._db.helpers._getPadObjects("View", padId); } diff --git a/server/src/export/geojson.ts b/server/src/export/geojson.ts index 84faba4e..d11f4d6d 100644 --- a/server/src/export/geojson.ts +++ b/server/src/export/geojson.ts @@ -1,9 +1,8 @@ -import { jsonStream, asyncIteratorToArray, streamPromiseToStream } from "../utils/streams.js"; +import { asyncIteratorToArray, streamPromiseToStream, jsonStreamArray, mapAsyncIterator, jsonStreamRecord, type JsonStream, concatAsyncIterators, flatMapAsyncIterator } from "../utils/streams.js"; import { compileExpression } from "facilmap-utils"; -import type { Marker, MarkerFeature, LineFeature, PadId } from "facilmap-types"; +import type { Marker, MarkerFeature, PadId, TrackPoint, Line } from "facilmap-types"; import Database from "../database/database.js"; import { cloneDeep, keyBy, mapValues, omit } from "lodash-es"; -import type { LineWithTrackPoints } from "../database/line.js"; import type { ReadableStream } from "stream/web"; export function exportGeoJson(database: Database, padId: PadId, filter?: string): ReadableStream { @@ -17,49 +16,41 @@ export function exportGeoJson(database: Database, padId: PadId, filter?: string) const types = keyBy(await asyncIteratorToArray(database.types.getTypes(padId)), "id"); - return jsonStream({ + return jsonStreamRecord({ type: "FeatureCollection", - ...(padData.defaultView ? { bbox: "%bbox%" } : { }), - facilmap: { - name: "%name%", - searchEngines: "%searchEngines%", - description: "%description%", - clusterMarkers: "%clusterMarkers", - views: "%views%", - types: "%types%" - }, - features: "%features%" - }, { - bbox: padData.defaultView && [padData.defaultView.left, padData.defaultView.bottom, padData.defaultView.right, padData.defaultView.top], - name: padData.name, - searchEngines: padData.searchEngines, - description: padData.description, - clusterMarkers: padData.clusterMarkers, - views: async function*() { - for await (const view of database.views.getViews(padId)) { - yield omit(view, ["id", "padId"]); - } - }, + ...(padData.defaultView ? { + bbox: [padData.defaultView.left, padData.defaultView.bottom, padData.defaultView.right, padData.defaultView.top] + } : { }), + facilmap: jsonStreamRecord({ + name: padData.name, + searchEngines: padData.searchEngines, + description: padData.description, + clusterMarkers: padData.clusterMarkers, + views: jsonStreamArray(mapAsyncIterator(database.views.getViews(padId), (view) => omit(view, ["id", "padId"]))) + }), types: mapValues(types, (type) => omit(type, ["id", "padId"])), - features: async function*() { - for await (const marker of database.markers.getPadMarkers(padId)) { + features: jsonStreamArray(concatAsyncIterators( + flatMapAsyncIterator(database.markers.getPadMarkers(padId), (marker) => { if (filterFunc(marker, types[marker.typeId])) { - yield markerToGeoJson(marker); + return [markerToGeoJson(marker)]; + } else { + return []; } - } - - for await (const line of database.lines.getPadLinesWithPoints(padId)) { + }), + flatMapAsyncIterator(database.lines.getPadLines(padId), (line) => { if (filterFunc(line, types[line.typeId])) { - yield lineToGeoJson(line); + return [lineToGeoJson(line, database.lines.getAllLinePoints(line.id))]; + } else { + return []; } - } - } + }) + )) }); })()); } -function markerToGeoJson(marker: Marker): MarkerFeature { - return { +function markerToGeoJson(marker: Marker): JsonStream { + return jsonStreamRecord({ type: "Feature", geometry: { type: "Point", @@ -74,16 +65,16 @@ function markerToGeoJson(marker: Marker): MarkerFeature { data: cloneDeep(marker.data), typeId: marker.typeId } - }; + } satisfies MarkerFeature); } -function lineToGeoJson(line: LineWithTrackPoints): LineFeature { - return { +function lineToGeoJson(line: Line, trackPoints: AsyncIterable): ReadableStream { + return jsonStreamRecord({ type: "Feature", - geometry: { + geometry: jsonStreamRecord({ type: "LineString", - coordinates: line.trackPoints.map((trackPoint) => [trackPoint.lon, trackPoint.lat]) - }, + coordinates: jsonStreamArray(mapAsyncIterator(trackPoints, (trackPoint) => [trackPoint.lon, trackPoint.lat])) + }), properties: { name: line.name, mode: line.mode, @@ -96,5 +87,5 @@ function lineToGeoJson(line: LineWithTrackPoints): LineFeature { routePoints: line.routePoints, typeId: line.typeId } - }; + }); } \ No newline at end of file diff --git a/server/src/export/gpx-line.ejs b/server/src/export/gpx-line.ejs deleted file mode 100644 index c8fd59e2..00000000 --- a/server/src/export/gpx-line.ejs +++ /dev/null @@ -1,24 +0,0 @@ - - - - <%=line.name || 'FacilMap route'%> - - - <<%=useTracks ? 'trk' : 'rte'%>> - <%=line.name || 'FacilMap route'%> -<% if(desc) { -%> - <%=desc%> -<% } -%> -<% if(useTracks) { -%> - -<% for(let trackPoint of line.trackPoints) { -%> - ele="<%=trackPoint.ele%>"<% } %> /> -<% } -%> - -<% } else { -%> -<% for(let routePoint of line.routePoints) { -%> - -<% } -%> -<% } -%> - > - \ No newline at end of file diff --git a/server/src/export/gpx.ts b/server/src/export/gpx.ts index 27314ba3..3bfe8d4d 100644 --- a/server/src/export/gpx.ts +++ b/server/src/export/gpx.ts @@ -1,14 +1,19 @@ -import { asyncIteratorToArray, asyncIteratorToStream } from "../utils/streams.js"; -import { compile } from "ejs"; +import { asyncIteratorToArray, asyncIteratorToStream, getZipEncodeStream, indentStream, stringToStream, type ZipEncodeStreamItem } from "../utils/streams.js"; import Database from "../database/database.js"; -import type { Field, PadId, Type } from "facilmap-types"; -import { compileExpression, normalizeLineName, normalizeMarkerName, normalizePadName, quoteHtml } from "facilmap-utils"; +import type { Field, Line, Marker, PadId, TrackPoint, Type } from "facilmap-types"; +import { compileExpression, getSafeFilename, normalizeLineName, normalizeMarkerName, normalizePadName, quoteHtml } from "facilmap-utils"; import type { LineWithTrackPoints } from "../database/line.js"; import { keyBy } from "lodash-es"; -import gpxLineEjs from "./gpx-line.ejs?raw"; import type { ReadableStream } from "stream/web"; -const lineTemplate = compile(gpxLineEjs); +const gpxHeader = ( + `\n` + + `` +); + +const gpxFooter = ( + `` +); const markerShapeToOsmand: Record = { "drop": "circle", @@ -34,6 +39,67 @@ function dataToText(fields: Field[], data: Record) { return text.join('\n\n'); } +function getMetadataGpx(data: { name: string }, extensions: Record = {}): string { + return ( + `\n` + + Object.entries({ + time: new Date().toISOString(), + ...data + }).map(([k, v]) => `\t<${quoteHtml(k)}>${quoteHtml(v)}\n`).join("") + + `` + + (Object.keys(extensions).length > 0 ? ( + `\n` + + `\n` + + Object.entries(extensions).map(([k, v]) => `\t<${quoteHtml(k)}>${quoteHtml(v)}\n`).join("") + + `` + ) : "") + ); +} + +function getMarkerGpx(marker: Marker, type: Type): ReadableStream { + const osmandBackground = markerShapeToOsmand[marker.shape || "drop"]; + return stringToStream( + `\n` + + `\t${quoteHtml(normalizeMarkerName(marker.name))}\n` + + `\t${quoteHtml(dataToText(type.fields, marker.data))}\n` + + `\t\n` + + (osmandBackground ? `\t\t${osmandBackground}\n` : "") + + `\t\t#${marker.colour}\n` + + `\t\n` + + `` + ); +} + +function getLineRouteGpx(line: LineForExport, type: Type | undefined): ReadableStream { + return stringToStream( + `\n` + + `\t${quoteHtml(normalizeLineName(line.name))}\n` + + (type ? `\t${quoteHtml(dataToText(type.fields, line.data ?? {}))}\n` : "") + + line.routePoints.map((routePoint) => ( + `\t\n` + )).join("") + + `` + ); +} + +function getLineTrackGpx(line: LineForExport, type: Type | undefined, trackPoints: AsyncIterable): ReadableStream { + return asyncIteratorToStream((async function*() { + yield ( + `\n` + + `\t${quoteHtml(normalizeLineName(line.name))}\n` + + (type ? `\t${quoteHtml(dataToText(type.fields, line.data ?? {}))}\n` : "") + + `\t\n` + ); + for await (const trackPoint of trackPoints) { + yield `\t\t\n`; + } + yield ( + `\t\n` + + `` + ); + })()); +} + export function exportGpx(database: Database, padId: PadId, useTracks: boolean, filter?: string): ReadableStream { return asyncIteratorToStream((async function* () { const filterFunc = compileExpression(filter); @@ -47,69 +113,150 @@ export function exportGpx(database: Database, padId: PadId, useTracks: boolean, throw new Error(`Pad ${padId} could not be found.`); yield ( - `\n` + - `\n` + - `\t\n` + - `\t\t${quoteHtml(normalizePadName(padData.name))}\n` + - `\t\t\n` + - `\t\n` + `${gpxHeader}\n` + + `\t${getMetadataGpx({ name: normalizePadName(padData.name) }).replaceAll("\n", "\n\t")}\n` ); for await (const marker of database.markers.getPadMarkers(padId)) { if (filterFunc(marker, types[marker.typeId])) { - const osmandBackground = markerShapeToOsmand[marker.shape || "drop"]; - yield ( - `\t\n` + - `\t\t${quoteHtml(normalizeMarkerName(marker.name))}\n` + - `\t\t${quoteHtml(dataToText(types[marker.typeId].fields, marker.data))}\n` + - `\t\t\n` + - (osmandBackground ? `\t\t\t${osmandBackground}\n` : "") + - `\t\t\t#${marker.colour}\n` + - `\t\t\n` + - `\t\n` - ); - } - } - - for await (const line of database.lines.getPadLinesWithPoints(padId)) { - if (filterFunc(line, types[line.typeId])) { - if (useTracks || line.mode == "track") { - yield ( - `\t\n` + - `\t\t${quoteHtml(normalizeLineName(line.name))}\n` + - `\t\t${dataToText(types[line.typeId].fields, line.data)}\n` + - `\t\t\n` + - line.trackPoints.map((trackPoint) => ( - `\t\t\t\n` - )).join("") + - `\t\t\n` + - `\t\n` - ); - } else { - yield ( - `\t\n` + - `\t\t${quoteHtml(normalizeLineName(line.name))}\n` + - `\t\t${quoteHtml(dataToText(types[line.typeId].fields, line.data))}\n` + - line.routePoints.map((routePoint) => ( - `\t\t\n` - )).join("") + - `\t\n` - ); + for await (const chunk of indentStream(getMarkerGpx(marker, types[marker.typeId]), { indent: "\t", indentFirst: true, addNewline: true })) { + yield chunk; } } } - yield ``; + for await (const line of database.lines.getPadLines(padId)) { + if (filterFunc(line, types[line.typeId])) { + if (useTracks || line.mode == "track") { + const trackPoints = database.lines.getAllLinePoints(line.id); + for await (const chunk of indentStream(getLineTrackGpx(line, types[line.typeId], trackPoints), { indent: "\t", indentFirst: true, addNewline: true })) { + yield chunk; + } + } else { + for await (const chunk of indentStream(getLineRouteGpx(line, types[line.typeId]), { indent: "\t", indentFirst: true, addNewline: true })) { + yield chunk; + } + } + } + } + + yield gpxFooter; })()); } -type LineForExport = Partial>; +export function exportGpxZip(database: Database, padId: PadId, useTracks: boolean, filter?: string): ReadableStream { + const encodeZipStream = getZipEncodeStream(); -export async function exportLineToGpx(line: LineForExport, type: Type | undefined, useTracks: boolean): Promise { - return lineTemplate({ - useTracks: (useTracks || line.mode == "track"), - time: new Date().toISOString(), - desc: type && dataToText(type.fields, line.data ?? {}), - line + asyncIteratorToStream((async function*(): AsyncIterable { + const filterFunc = compileExpression(filter); + + const [padData, types] = await Promise.all([ + database.pads.getPadData(padId), + asyncIteratorToArray(database.types.getTypes(padId)).then((types) => keyBy(types, 'id')) + ]); + + if (!padData) { + throw new Error(`Pad ${padId} could not be found.`); + } + + yield { + filename: "markers.gpx", + data: asyncIteratorToStream((async function*() { + yield ( + `${gpxHeader}\n` + + `\t${getMetadataGpx({ name: normalizePadName(padData.name) }).replaceAll("\n", "\n\t")}\n` + ); + + for await (const marker of database.markers.getPadMarkers(padId)) { + if (filterFunc(marker, types[marker.typeId])) { + for await (const chunk of indentStream(getMarkerGpx(marker, types[marker.typeId]), { indent: "\t", indentFirst: true, addNewline: true })) { + yield chunk; + } + } + } + + yield gpxFooter; + })()) + }; + + yield { + filename: "lines/", + data: null + }; + + const names = new Set(); + + for await (const line of database.lines.getPadLines(padId)) { + if (filterFunc(line, types[line.typeId])) { + const lineName = normalizeLineName(line.name); + let name = lineName; + for (let i = 1; names.has(name); i++) { + name = `${lineName} (${i})`; + } + names.add(name); + + const filename = `lines/${getSafeFilename(name)}.gpx`; + + if (useTracks || line.mode == "track") { + const trackPoints = database.lines.getAllLinePoints(line.id); + yield { + filename, + data: exportLineToTrackGpx(line, types[line.typeId], trackPoints) + }; + } else { + yield { + filename, + data: exportLineToRouteGpx(line, types[line.typeId]) + }; + } + } + } + })()).pipeTo(encodeZipStream.writable); + + return encodeZipStream.readable; +} + +type LineForExport = Pick & Partial>; + +function getLineMetadataGpx(line: LineForExport): string { + return getMetadataGpx({ + name: normalizeLineName(line.name) + }, { + ...(line.colour ? { + color: `#${line.colour}` + } : {}), + ...(line.width ? { + width: `${line.width}` + } : {}) }); } + +export function exportLineToTrackGpx(line: LineForExport, type: Type | undefined, trackPoints: AsyncIterable): ReadableStream { + return asyncIteratorToStream((async function*() { + yield ( + `${gpxHeader}\n` + + `\t${getLineMetadataGpx(line).replaceAll("\n", "\n\t")}\n` + ); + + for await (const chunk of indentStream(getLineTrackGpx(line, type, trackPoints), { indent: "\t", indentFirst: true, addNewline: true })) { + yield chunk; + } + + yield gpxFooter; + })()); +} + +export function exportLineToRouteGpx(line: LineForExport, type: Type | undefined): ReadableStream { + return asyncIteratorToStream((async function*() { + yield ( + `${gpxHeader}\n` + + `\t${getLineMetadataGpx(line).replaceAll("\n", "\n\t")}\n` + ); + + for await (const chunk of indentStream(getLineRouteGpx(line, type), { indent: "\t", indentFirst: true, addNewline: true })) { + yield chunk; + } + + yield gpxFooter; + })()); +} diff --git a/server/src/frontend.ts b/server/src/frontend.ts index 31687b34..49f25df1 100644 --- a/server/src/frontend.ts +++ b/server/src/frontend.ts @@ -7,7 +7,7 @@ import { Router, type RequestHandler } from "express"; import { static as expressStatic } from "express"; import { normalizePadName, type InjectedConfig, quoteHtml } from "facilmap-utils"; import config from "./config"; -import { asyncIteratorToArray, jsonStream, streamPromiseToStream, streamReplace } from "./utils/streams"; +import { streamPromiseToStream, streamReplace } from "./utils/streams"; import { ReadableStream } from "stream/web"; import { generateRandomId } from "./utils/utils"; import type { TableParams } from "./export/table"; @@ -143,10 +143,7 @@ export async function getStaticFrontendMiddleware(): Promise { export async function getPwaManifest(): Promise { const template = await readFile(paths.pwaManifest).then((t) => t.toString()); - const chunks = await asyncIteratorToArray(jsonStream(JSON.parse(template), { - APP_NAME: config.appName - })); - return chunks.join(""); + return template.replaceAll("%APP_NAME%", config.appName); } export async function getOpensearchXml(baseUrl: string): Promise { diff --git a/server/src/shims.d.ts b/server/src/shims.d.ts index 3dcced3b..9fe2a205 100644 --- a/server/src/shims.d.ts +++ b/server/src/shims.d.ts @@ -1,4 +1,6 @@ declare module "*?raw" { declare const content: string; export default content; -} \ No newline at end of file +} + +declare module "zip-stream"; \ No newline at end of file diff --git a/server/src/socket/socket-v2.ts b/server/src/socket/socket-v2.ts index ef901860..be82cdee 100644 --- a/server/src/socket/socket-v2.ts +++ b/server/src/socket/socket-v2.ts @@ -1,8 +1,8 @@ import { promiseProps, type PromiseMap } from "../utils/utils.js"; -import { asyncIteratorToArray } from "../utils/streams.js"; +import { asyncIteratorToArray, streamToString } from "../utils/streams.js"; import { isInBbox } from "../utils/geo.js"; import { Socket, type Socket as SocketIO } from "socket.io"; -import { exportLineToGpx } from "../export/gpx.js"; +import { exportLineToRouteGpx, exportLineToTrackGpx } from "../export/gpx.js"; import { find } from "../search.js"; import { geoipLookup } from "../geoip.js"; import { cloneDeep, isEqual, omit } from "lodash-es"; @@ -254,7 +254,7 @@ export class SocketConnectionV2 extends SocketConnection { if (data.mode != "track") { for (const route of [...(this.route ? [this.route] : []), ...Object.values(this.routes)]) { if(isEqual(route.routePoints, data.routePoints) && data.mode == route.mode) { - fromRoute = { ...route, trackPoints: await this.database.routes.getAllRoutePoints(route.id) }; + fromRoute = { ...route, trackPoints: await asyncIteratorToArray(this.database.routes.getAllRoutePoints(route.id)) }; break; } } @@ -273,7 +273,7 @@ export class SocketConnectionV2 extends SocketConnection { if (data.mode != "track") { for (const route of [...(this.route ? [this.route] : []), ...Object.values(this.routes)]) { if(isEqual(route.routePoints, data.routePoints) && data.mode == route.mode) { - fromRoute = { ...route, trackPoints: await this.database.routes.getAllRoutePoints(route.id) }; + fromRoute = { ...route, trackPoints: await asyncIteratorToArray(this.database.routes.getAllRoutePoints(route.id)) }; break; } } @@ -300,19 +300,16 @@ export class SocketConnectionV2 extends SocketConnection { const lineP = this.database.lines.getLine(this.padId, data.id); lineP.catch(() => null); // Avoid unhandled promise error (https://stackoverflow.com/a/59062117/242365) - const [line, trackPoints, type] = await Promise.all([ + const [line, type] = await Promise.all([ lineP, - this.database.lines.getAllLinePoints(data.id), lineP.then((line) => this.database.types.getType(this.padId as string, line.typeId)) ]); - const lineWithTrackPoints = { ...line, trackPoints }; - switch(data.format) { case "gpx-trk": - return exportLineToGpx(lineWithTrackPoints, type, true); + return await streamToString(exportLineToTrackGpx(line, type, this.database.lines.getAllLinePoints(line.id))); case "gpx-rte": - return exportLineToGpx(lineWithTrackPoints, type, false); + return await streamToString(exportLineToRouteGpx(line, type)); default: throw new Error("Unknown format."); } @@ -536,15 +533,13 @@ export class SocketConnectionV2 extends SocketConnection { throw new Error("Route not available."); } - const trackPoints = await this.database.routes.getAllRoutePoints(route.id); - - const routeInfo = { ...this.route, trackPoints }; + const routeInfo = { ...route, name: "FacilMap route", data: {} }; switch(data.format) { case "gpx-trk": - return await exportLineToGpx(routeInfo, undefined, true); + return await streamToString(exportLineToTrackGpx(routeInfo, undefined, this.database.routes.getAllRoutePoints(route.id))); case "gpx-rte": - return await exportLineToGpx(routeInfo, undefined, false); + return await streamToString(exportLineToRouteGpx(routeInfo, undefined)); default: throw new Error("Unknown format."); } diff --git a/server/src/utils/__tests__/streams.test.ts b/server/src/utils/__tests__/streams.test.ts index 1c4d784a..89837f3d 100644 --- a/server/src/utils/__tests__/streams.test.ts +++ b/server/src/utils/__tests__/streams.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { arrayToAsyncIterator, asyncIteratorToArray, asyncIteratorToStream, jsonStream, streamPromiseToStream, streamReplace } from "../streams.js"; +import { arrayToAsyncIterator, asyncIteratorToArray, asyncIteratorToStream, jsonStreamArray, jsonStreamRecord, streamPromiseToStream, streamReplace } from "../streams.js"; import { ReadableStream } from "stream/web"; describe("streamPromiseToStream", () => { @@ -48,64 +48,60 @@ describe("streamPromiseToStream", () => { }); test('jsonStream', async () => { - const template = { - test1: "%var1%", - test2: { + const stream = jsonStreamRecord({ + test1: { test: 'object' }, + test2: jsonStreamRecord({ one: "one", - two: "%var2%", + two: jsonStreamArray([ { object: 'one' }, { object: 'two' } ]), three: "three" - }, - test3: "%var3%", - test4: "%var4%", - test5: "%var5%", - test6: "%var6%", - test7: "%var7%", - test8: "bla" - }; - - const stream = jsonStream(template, { - var1: { test: 'object' }, - var2: arrayToAsyncIterator([ { object: 'one' }, { object: 'two' } ]), - var3: () => arrayToAsyncIterator([ { object: 'one' }, { object: 'two' } ]), - var4: 'asdf', - var5: () => 'bla', - var6: Promise.resolve('promise'), - var7: () => Promise.resolve('async') + }), + test3: jsonStreamRecord(arrayToAsyncIterator(Object.entries({ + one: "one", + two: jsonStreamArray(arrayToAsyncIterator([ { object: 'one' }, { object: 'two' } ])), + three: "three" + }))), + test4: jsonStreamArray([ + "one", + jsonStreamRecord({ object1: "one", object2: "two" }), + "three" + ]), + test5: jsonStreamArray(arrayToAsyncIterator([ + "one", + jsonStreamRecord(arrayToAsyncIterator(Object.entries({ object1: "one", object2: "two" }))), + "three" + ])), + test6: Promise.resolve("promise"), + test7: "string" }); const result = (await asyncIteratorToArray(stream as any)).join(""); - expect(result).toBe( -`{ - "test1": { - "test": "object" - }, - "test2": { - "one": "one", - "two": [ - { - "object": "one" - }, - { - "object": "two" - } - ], - "three": "three" - }, - "test3": [ - { - "object": "one" + expect(result).toBe(JSON.stringify({ + test1: { + test: "object" }, - { - "object": "two" - } - ], - "test4": "asdf", - "test5": "bla", - "test6": "promise", - "test7": "async", - "test8": "bla" -}` - ); + test2: { + one: "one", + two: [{ object: "one" }, { object: "two" }], + three: "three" + }, + test3: { + one: "one", + two: [{ object: "one" }, { object: "two" }], + three: "three" + }, + test4: [ + "one", + { object1: "one", object2: "two" }, + "three" + ], + test5: [ + "one", + { object1: "one", object2: "two" }, + "three" + ], + test6: "promise", + test7: "string" + }, undefined, "\t")); }); test("streamReplace", async () => { diff --git a/server/src/utils/streams.ts b/server/src/utils/streams.ts index 09eff6fc..5097463a 100644 --- a/server/src/utils/streams.ts +++ b/server/src/utils/streams.ts @@ -1,4 +1,6 @@ -import { ReadableStream, TransformStream } from "stream/web"; +import { Readable } from "stream"; +import { type QueuingStrategy, ReadableStream, TransformStream } from "stream/web"; +import Packer from "zip-stream"; export async function asyncIteratorToArray(iterator: AsyncIterable): Promise> { const result: T[] = []; @@ -8,23 +10,48 @@ export async function asyncIteratorToArray(iterator: AsyncIterable): Promi return result; } -export async function* arrayToAsyncIterator(array: T[]): AsyncGenerator { +export async function* arrayToAsyncIterator(array: T[]): AsyncIterable { for (const it of array) { yield it; } } -export function asyncIteratorToStream(iterator: AsyncGenerator): ReadableStream { +export function asyncIteratorToStream(iterator: AsyncIterable, strategy?: QueuingStrategy): ReadableStream { + const it = iterator[Symbol.asyncIterator](); return new ReadableStream({ async pull(controller) { - const { value, done } = await iterator.next(); + const { value, done } = await it.next(); if (done) { controller.close(); } else { controller.enqueue(value); } }, - }); + }, strategy); +} + +export function mapAsyncIterator(iterator: AsyncIterable, mapper: (it: T) => O): AsyncIterable { + return flatMapAsyncIterator(iterator, (it) => [mapper(it)]); +} + +export function filterAsyncIterator(iterator: AsyncIterable, filter: (it: T) => boolean): AsyncIterable { + return flatMapAsyncIterator(iterator, (it) => filter(it) ? [it] : []); +} + +export async function* flatMapAsyncIterator(iterator: AsyncIterable, mapper: (it: T) => O[]): AsyncIterable { + for await (const it of iterator) { + for (const o of mapper(it)) { + yield o; + } + } +} + +export async function* concatAsyncIterators(...iterators: Array>): AsyncIterable { + for (const iterator of iterators) { + for await (const it of iterator) { + yield it; + } + } } export function streamPromiseToStream(streamPromise: Promise>): ReadableStream { @@ -57,40 +84,56 @@ export function flatMapStream(stream: ReadableStream, mapper: (it: T) = return transform.readable; } -export function jsonStream(template: any, data: Record | Promise | any | (() => AsyncGenerator | Promise | any)>): ReadableStream { - return asyncIteratorToStream((async function*() { - let lastIndent = ''; +type Stringifiable = boolean | number | string | undefined | null | Array | /* Should be Record here, but that does not work */ object; +const jsonStreamSymbol = Symbol("jsonStream"); +export type JsonStream = ReadableStream & { [jsonStreamSymbol]: true }; +const isJsonStream = (obj: unknown): obj is JsonStream => !!obj && typeof obj === "object" && jsonStreamSymbol in obj && !!obj[jsonStreamSymbol]; - const parts = JSON.stringify(template, undefined, "\t").split(/"%([a-zA-Z0-9-_]+)%"/); - for (let i = 0; i < parts.length; i++) { - const part = parts[i]; +export function jsonStreamArray(iterator: Iterable | AsyncIterable): JsonStream { + return Object.assign(asyncIteratorToStream((async function*() { + let first = true; + yield "["; + for await (const value of iterator) { + const prefix = `${first ? "" : ","}\n\t`; + first = false; - if (i % 2 == 0) { - const lastLineBreak = part.lastIndexOf('\n'); - if (lastLineBreak != -1) - lastIndent = part.slice(lastLineBreak + 1).match(/^(\t*)/)![1]; - - yield part; - } else { - const value = await (typeof data[part] === 'function' ? data[part]() : data[part]); - - if (typeof value === 'object' && value && Symbol.asyncIterator in value) { - let first = true; - const indent = lastIndent + "\t"; - yield '[\n'; - for await (const obj of value) { - const prefix = first ? '' : ',\n'; - first = false; - yield prefix + JSON.stringify(obj, undefined, "\t").replace(/^/gm, indent); - } - yield '\n' + lastIndent + ']'; - } else { - const indent = lastIndent; - yield JSON.stringify(value, undefined, "\t").replace(/\n/g, '\n' + indent); + if (isJsonStream(value)) { + yield prefix; + for await (const chunk of streamReplace(value, { "\n": `\n\t` })) { + yield chunk; } + } else { + yield `${prefix}${JSON.stringify(value, undefined, "\t").replaceAll("\n", "\n\t")}`; } } - })()); + yield `${first ? "" : "\n"}]`; + })()), { + [jsonStreamSymbol]: true as const + }) +} + +export function jsonStreamRecord(iterator: Record | JsonStream> | AsyncIterable<[key: string | number, value: Stringifiable | Promise | JsonStream]>): JsonStream { + return Object.assign(asyncIteratorToStream((async function*() { + let first = true; + yield "{"; + const normalizedIterator = Symbol.asyncIterator in iterator ? iterator : Object.entries(iterator); + for await (const [key, value] of normalizedIterator) { + const prefix = `${first ? "" : ","}\n\t${JSON.stringify(key)}: `; + first = false; + + if (isJsonStream(value)) { + yield prefix; + for await (const chunk of streamReplace(value, { "\n": `\n\t` })) { + yield chunk; + } + } else { + yield `${prefix}${JSON.stringify(await value, undefined, "\t").replaceAll("\n", "\n\t")}`; + } + } + yield `${first ? "" : "\n"}}`; + })()), { + [jsonStreamSymbol]: true as const + }) } export function stringToStream(string: string): ReadableStream { @@ -99,6 +142,10 @@ export function stringToStream(string: string): ReadableStream { })()); } +export async function streamToString(stream: ReadableStream): Promise { + return (await asyncIteratorToArray(stream)).join(""); +} + export function streamReplace(stream: ReadableStream | string, replace: Record | string>): ReadableStream { const normalizedStream = typeof stream === "string" ? stringToStream(stream) : stream; @@ -159,3 +206,47 @@ export function streamReplace(stream: ReadableStream | string, replace: } })()); } + +export type ZipEncodeStreamItem = { filename: string, data: ReadableStream | null }; + +export function getZipEncodeStream(): TransformStream { + const archive = new Packer(); + + const writable = new WritableStream({ + async write(chunk) { + await new Promise((resolve, reject) => { + archive.entry(chunk.data && Readable.fromWeb(chunk.data), { name: chunk.filename }, (err: any) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + }, + + close() { + archive.finish(); + } + }); + + return { + readable: Readable.toWeb(archive), + writable + }; +} + +export function indentStream(stream: ReadableStream, { indent, indentFirst, addNewline }: { indent: string, indentFirst: boolean; addNewline: boolean }): ReadableStream { + return asyncIteratorToStream((async function*() { + let first = true; + for await (const chunk of streamReplace(stream, { "\n": `\n${indent}` })) { + if (chunk.length > 0) { + yield `${first && indentFirst ? indent : ""}${chunk}`; + first = false; + } + } + if (addNewline && !first) { + yield "\n"; + } + })()); +} \ No newline at end of file diff --git a/server/src/utils/utils.ts b/server/src/utils/utils.ts index 02a1295a..24ca108e 100644 --- a/server/src/utils/utils.ts +++ b/server/src/utils/utils.ts @@ -38,4 +38,4 @@ export async function fileExists(filename: string): Promise { throw err; } } -} +} \ No newline at end of file diff --git a/server/src/webserver.ts b/server/src/webserver.ts index 5f5fcb3d..cc115399 100644 --- a/server/src/webserver.ts +++ b/server/src/webserver.ts @@ -5,11 +5,11 @@ import { stringifiedIdValidator, type PadId } from "facilmap-types"; import { createSingleTable, createTable } from "./export/table.js"; import Database from "./database/database"; import { exportGeoJson } from "./export/geojson.js"; -import { exportGpx } from "./export/gpx.js"; +import { exportGpx, exportGpxZip } from "./export/gpx.js"; import domainMiddleware from "express-domain-middleware"; import { Readable, Writable } from "stream"; import { getOpensearchXml, getPwaManifest, getStaticFrontendMiddleware, renderMap, type RenderMapParams } from "./frontend"; -import { normalizePadName } from "facilmap-utils"; +import { getSafeFilename, normalizePadName } from "facilmap-utils"; import { paths } from "facilmap-frontend/build.js"; import config from "./config"; import { exportCsv } from "./export/csv.js"; @@ -104,10 +104,26 @@ export async function initWebserver(database: Database, port: number, host?: str throw new Error(`Map with ID ${req.params.padId} could not be found.`); res.set("Content-type", "application/gpx+xml"); - res.attachment(padData.name.replace(/[\\/:*?"<>|]+/g, '_') + ".gpx"); + res.attachment(`${getSafeFilename(normalizePadName(padData.name))}.gpx`); exportGpx(database, padData ? padData.id : req.params.padId, query.useTracks == "1", query.filter).pipeTo(Writable.toWeb(res)); }); + app.get("/:padId/gpx/zip", async (req: Request<{ padId: string }>, res: Response) => { + const query = z.object({ + useTracks: z.enum(["0", "1"]).default("0"), + filter: z.string().optional() + }).parse(req.query); + + const padData = await database.pads.getPadDataByAnyId(req.params.padId); + + if(!padData) + throw new Error(`Map with ID ${req.params.padId} could not be found.`); + + res.set("Content-type", "application/zip"); + res.attachment(padData.name.replace(/[\\/:*?"<>|]+/g, '_') + ".zip"); + exportGpxZip(database, padData ? padData.id : req.params.padId, query.useTracks == "1", query.filter).pipeTo(Writable.toWeb(res)); + }); + app.get("/:padId/table", async (req: Request<{ padId: string }>, res: Response) => { const query = z.object({ filter: z.string().optional(), diff --git a/utils/src/utils.ts b/utils/src/utils.ts index 3b645aad..ef56d3d6 100644 --- a/utils/src/utils.ts +++ b/utils/src/utils.ts @@ -3,12 +3,16 @@ import { cloneDeep, isEqual } from "lodash-es"; const LETTERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; const LENGTH = 12; -export function quoteJavaScript(str: any): string { - return "'" + `${str}`.replace(/['\\]/g, '\\$1').replace(/\n/g, "\\n") + "'"; -} - -export function quoteHtml(str: any): string { - return `${str}`.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """).replace(/'/g, "'"); +export function quoteHtml(str: string | number): string { + return `${str}` + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'") + .replace(/\n/g, " ") + .replace(/\r/g, " ") + .replace(/\t/g, " "); } export function quoteRegExp(str: string): string { @@ -159,3 +163,7 @@ export function mergeObject>(oldObject: T | und targetObject[i] = cloneDeep(newObject[i]); } } + +export function getSafeFilename(fname: string): string { + return fname.replace(/[\\/:*?"<>|]+/g, '_'); +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 2e9d2dee..422be864 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1686,6 +1686,15 @@ __metadata: languageName: node linkType: hard +"abort-controller@npm:^3.0.0": + version: 3.0.0 + resolution: "abort-controller@npm:3.0.0" + dependencies: + event-target-shim: ^5.0.0 + checksum: 170bdba9b47b7e65906a28c8ce4f38a7a369d78e2271706f020849c1bfe0ee2067d4261df8bbb66eb84f79208fd5b710df759d64191db58cfba7ce8ef9c54b75 + languageName: node + linkType: hard + "accepts@npm:~1.3.4, accepts@npm:~1.3.5, accepts@npm:~1.3.7": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -1816,6 +1825,20 @@ __metadata: languageName: node linkType: hard +"archiver-utils@npm:^5.0.0": + version: 5.0.1 + resolution: "archiver-utils@npm:5.0.1" + dependencies: + glob: ^10.0.0 + graceful-fs: ^4.2.0 + lazystream: ^1.0.0 + lodash: ^4.17.15 + normalize-path: ^3.0.0 + readable-stream: ^3.6.0 + checksum: a6e907cea41486ff95fdbe1b267890df96d51e2712b6604cb9b0c5f7348aea475091f41e0e6fced3a893490deddfd8d0704f66c8aa470394cff3618bbec6d4cc + languageName: node + linkType: hard + "argparse@npm:^2.0.1": version: 2.0.1 resolution: "argparse@npm:2.0.1" @@ -1979,6 +2002,13 @@ __metadata: languageName: node linkType: hard +"base64-js@npm:^1.3.1": + version: 1.5.1 + resolution: "base64-js@npm:1.5.1" + checksum: 669632eb3745404c2f822a18fc3a0122d2f9a7a13f7fb8b5823ee19d1d2ff9ee5b52c53367176ea4ad093c332fd5ab4bd0ebae5a8e27917a4105a4cfc86b1005 + languageName: node + linkType: hard + "base64id@npm:2.0.0, base64id@npm:~2.0.0": version: 2.0.0 resolution: "base64id@npm:2.0.0" @@ -2109,6 +2139,16 @@ __metadata: languageName: node linkType: hard +"buffer@npm:^6.0.3": + version: 6.0.3 + resolution: "buffer@npm:6.0.3" + dependencies: + base64-js: ^1.3.1 + ieee754: ^1.2.1 + checksum: 5ad23293d9a731e4318e420025800b42bf0d264004c0286c8cc010af7a270c7a0f6522e84f54b9ad65cbd6db20b8badbfd8d2ebf4f80fa03dab093b89e68c3f9 + languageName: node + linkType: hard + "buffers@npm:~0.1.1": version: 0.1.1 resolution: "buffers@npm:0.1.1" @@ -2353,6 +2393,18 @@ __metadata: languageName: node linkType: hard +"compress-commons@npm:^6.0.0": + version: 6.0.1 + resolution: "compress-commons@npm:6.0.1" + dependencies: + crc-32: ^1.2.0 + crc32-stream: ^6.0.0 + normalize-path: ^3.0.0 + readable-stream: ^4.0.0 + checksum: a2ecd7c536cb4e1f029863464202e584ef4ea1b1f6a22da526efaf744ba6b59d29e860a637c046118cc38559f0b202f1d4dfa5c445a8c0f4bbf54eabcbd74bbf + languageName: node + linkType: hard + "compressible@npm:~2.0.16": version: 2.0.18 resolution: "compressible@npm:2.0.18" @@ -2517,6 +2569,25 @@ __metadata: languageName: node linkType: hard +"crc-32@npm:^1.2.0": + version: 1.2.2 + resolution: "crc-32@npm:1.2.2" + bin: + crc32: bin/crc32.njs + checksum: ad2d0ad0cbd465b75dcaeeff0600f8195b686816ab5f3ba4c6e052a07f728c3e70df2e3ca9fd3d4484dc4ba70586e161ca5a2334ec8bf5a41bf022a6103ff243 + languageName: node + linkType: hard + +"crc32-stream@npm:^6.0.0": + version: 6.0.0 + resolution: "crc32-stream@npm:6.0.0" + dependencies: + crc-32: ^1.2.0 + readable-stream: ^4.0.0 + checksum: e6edc2f81bc387daef6d18b2ac18c2ffcb01b554d3b5c7d8d29b177505aafffba574658fdd23922767e8dab1183d1962026c98c17e17fb272794c33293ef607c + languageName: node + linkType: hard + "cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.2, cross-spawn@npm:^7.0.3": version: 7.0.3 resolution: "cross-spawn@npm:7.0.3" @@ -3467,6 +3538,20 @@ __metadata: languageName: node linkType: hard +"event-target-shim@npm:^5.0.0": + version: 5.0.1 + resolution: "event-target-shim@npm:5.0.1" + checksum: 1ffe3bb22a6d51bdeb6bf6f7cf97d2ff4a74b017ad12284cc9e6a279e727dc30a5de6bb613e5596ff4dc3e517841339ad09a7eec44266eccb1aa201a30448166 + languageName: node + linkType: hard + +"events@npm:^3.3.0": + version: 3.3.0 + resolution: "events@npm:3.3.0" + checksum: f6f487ad2198aa41d878fa31452f1a3c00958f46e9019286ff4787c84aac329332ab45c9cdc8c445928fc6d7ded294b9e005a7fce9426488518017831b272780 + languageName: node + linkType: hard + "execa@npm:^8.0.1": version: 8.0.1 resolution: "execa@npm:8.0.1" @@ -3734,6 +3819,7 @@ __metadata: vite-plugin-dts: ^3.7.3 vite-tsconfig-paths: ^4.3.1 vitest: ^1.3.1 + zip-stream: ^6.0.0 zod: ^3.22.4 languageName: unknown linkType: soft @@ -4174,7 +4260,7 @@ __metadata: languageName: node linkType: hard -"glob@npm:^10.2.2, glob@npm:^10.3.10, glob@npm:^10.3.7": +"glob@npm:^10.0.0, glob@npm:^10.2.2, glob@npm:^10.3.10, glob@npm:^10.3.7": version: 10.3.10 resolution: "glob@npm:10.3.10" dependencies: @@ -4264,7 +4350,7 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.10, graceful-fs@npm:^4.2.2, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6": +"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.10, graceful-fs@npm:^4.2.2, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" checksum: ac85f94da92d8eb6b7f5a8b20ce65e43d66761c55ce85ac96df6865308390da45a8d3f0296dd3a663de65d30ba497bd46c696cc1e248c72b13d6d567138a4fc7 @@ -4465,7 +4551,7 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:^1.1.12": +"ieee754@npm:^1.1.12, ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" checksum: 5144c0c9815e54ada181d80a0b810221a253562422e7c6c3a60b1901154184f49326ec239d618c416c1c5945a2e197107aee8d986a3dd836b53dffefd99b5e7e @@ -5017,6 +5103,15 @@ __metadata: languageName: node linkType: hard +"lazystream@npm:^1.0.0": + version: 1.0.1 + resolution: "lazystream@npm:1.0.1" + dependencies: + readable-stream: ^2.0.5 + checksum: 822c54c6b87701a6491c70d4fabc4cafcf0f87d6b656af168ee7bb3c45de9128a801cb612e6eeeefc64d298a7524a698dd49b13b0121ae50c2ae305f0dcc5310 + languageName: node + linkType: hard + "leaflet-auto-graticule@npm:^2.0.0": version: 2.0.0 resolution: "leaflet-auto-graticule@npm:2.0.0" @@ -5199,7 +5294,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.21, lodash@npm:~4.17.15": +"lodash@npm:^4.17.15, lodash@npm:^4.17.21, lodash@npm:~4.17.15": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 @@ -6414,6 +6509,13 @@ __metadata: languageName: node linkType: hard +"process@npm:^0.11.10": + version: 0.11.10 + resolution: "process@npm:0.11.10" + checksum: bfcce49814f7d172a6e6a14d5fa3ac92cc3d0c3b9feb1279774708a719e19acd673995226351a082a9ae99978254e320ccda4240ddc474ba31a76c79491ca7c3 + languageName: node + linkType: hard + "promise-retry@npm:^2.0.1": version: 2.0.1 resolution: "promise-retry@npm:2.0.1" @@ -6523,7 +6625,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.2, readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -6538,7 +6640,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^3.0.2": +"readable-stream@npm:^3.0.2, readable-stream@npm:^3.6.0": version: 3.6.2 resolution: "readable-stream@npm:3.6.2" dependencies: @@ -6549,6 +6651,19 @@ __metadata: languageName: node linkType: hard +"readable-stream@npm:^4.0.0": + version: 4.5.2 + resolution: "readable-stream@npm:4.5.2" + dependencies: + abort-controller: ^3.0.0 + buffer: ^6.0.3 + events: ^3.3.0 + process: ^0.11.10 + string_decoder: ^1.3.0 + checksum: c4030ccff010b83e4f33289c535f7830190773e274b3fcb6e2541475070bdfd69c98001c3b0cb78763fc00c8b62f514d96c2b10a8bd35d5ce45203a25fa1d33a + languageName: node + linkType: hard + "readdirp@npm:~3.6.0": version: 3.6.0 resolution: "readdirp@npm:3.6.0" @@ -7303,7 +7418,7 @@ __metadata: languageName: node linkType: hard -"string_decoder@npm:^1.1.1": +"string_decoder@npm:^1.1.1, string_decoder@npm:^1.3.0": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" dependencies: @@ -8427,6 +8542,17 @@ __metadata: languageName: node linkType: hard +"zip-stream@npm:^6.0.0": + version: 6.0.0 + resolution: "zip-stream@npm:6.0.0" + dependencies: + archiver-utils: ^5.0.0 + compress-commons: ^6.0.0 + readable-stream: ^4.0.0 + checksum: 114565901579dc023b41b2d3d3ce2721ad734032295d50977dd58899cb6d1922abb6336ee388f6f90b74e0a384cb9c590f80cf9f1ac9ede865bd212ba96f6070 + languageName: node + linkType: hard + "zod@npm:^3.22.4": version: 3.22.4 resolution: "zod@npm:3.22.4"