From 39b7eca2a14a10a022d3bcfd4cd4f7912123cef0 Mon Sep 17 00:00:00 2001 From: Mike Barry Date: Tue, 19 Oct 2021 21:57:47 -0400 Subject: [PATCH] CI and docs --- .github/ISSUE_TEMPLATE/bug_report.md | 32 + .github/ISSUE_TEMPLATE/feature_request.md | 20 + .github/cache-sources-action/action.yml | 22 + .github/dependabot.yml | 33 + .github/workflows/docs.yml | 21 + .github/workflows/docs_mlc_config.json | 13 + .github/workflows/maven.yml | 101 +- .github/workflows/performance.yml | 92 + .github/workflows/release.yml | 74 + .github/workflows/snapshot.yml | 42 + .github/workflows/update-pr.yml | 89 + .gitignore | 2 + .idea/codeStyles/Project.xml | 6 + .mvn/wrapper/MavenWrapperDownloader.java | 117 + .mvn/wrapper/maven-wrapper.jar | Bin 0 -> 50710 bytes .mvn/wrapper/maven-wrapper.properties | 2 + ARCHITECTURE.md | 119 + CONTRIBUTING.md | 35 + NOTICE.md | 10 +- PLANET.md | 142 + README.md | 226 + config-example.properties | 13 +- diagrams/architecture.png | Bin 0 -> 243706 bytes diagrams/architecture.puml | 107 + diagrams/demo.png | Bin 0 -> 499806 bytes diagrams/slicing.png | Bin 0 -> 9002 bytes flatmap-basemap/README.md | 48 + flatmap-basemap/pom.xml | 57 + .../flatmap/basemap/BasemapMain.java | 24 +- .../flatmap/basemap/BasemapProfile.java | 15 +- .../onthegomap/flatmap/basemap}/Generate.java | 47 +- .../onthegomap/flatmap/basemap}/Layer.java | 4 +- .../generated/OpenMapTilesSchema.java | 44 +- .../flatmap/basemap}/generated/Tables.java | 2 +- .../basemap}/layers/AerodromeLabel.java | 12 +- .../flatmap/basemap}/layers/Aeroway.java | 6 +- .../flatmap/basemap}/layers/Boundary.java | 24 +- .../flatmap/basemap}/layers/Building.java | 14 +- .../flatmap/basemap}/layers/Housenumber.java | 6 +- .../flatmap/basemap}/layers/Landcover.java | 12 +- .../flatmap/basemap}/layers/Landuse.java | 14 +- .../flatmap/basemap}/layers/MountainPeak.java | 16 +- .../flatmap/basemap}/layers/Park.java | 16 +- .../flatmap/basemap}/layers/Place.java | 20 +- .../flatmap/basemap}/layers/Poi.java | 20 +- .../basemap}/layers/Transportation.java | 17 +- .../basemap}/layers/TransportationName.java | 32 +- .../flatmap/basemap}/layers/Water.java | 14 +- .../flatmap/basemap}/layers/WaterName.java | 16 +- .../flatmap/basemap}/layers/Waterway.java | 18 +- .../flatmap/basemap}/util/LanguageUtils.java | 6 +- .../flatmap/basemap}/util/Utils.java | 2 +- .../flatmap/basemap/util/VerifyMonaco.java | 44 + .../flatmap/basemap/BasemapProfileTest.java | 6 +- .../flatmap/basemap/BasemapTest.java | 34 +- .../flatmap/basemap}/GenerateTest.java | 4 +- .../basemap}/layers/AbstractLayerTest.java | 14 +- .../basemap}/layers/AerodromeLabelTest.java | 2 +- .../flatmap/basemap}/layers/AerowayTest.java | 2 +- .../flatmap/basemap}/layers/BoundaryTest.java | 6 +- .../flatmap/basemap}/layers/BuildingTest.java | 4 +- .../basemap}/layers/HousenumberTest.java | 2 +- .../basemap}/layers/LandcoverTest.java | 4 +- .../flatmap/basemap}/layers/LanduseTest.java | 4 +- .../basemap}/layers/MountainPeakTest.java | 2 +- .../flatmap/basemap}/layers/ParkTest.java | 2 +- .../flatmap/basemap}/layers/PlaceTest.java | 8 +- .../flatmap/basemap}/layers/PoiTest.java | 2 +- .../basemap}/layers/TransportationTest.java | 6 +- .../basemap}/layers/WaterNameTest.java | 8 +- .../flatmap/basemap}/layers/WaterTest.java | 8 +- .../flatmap/basemap}/layers/WaterwayTest.java | 4 +- .../basemap}/util/LanguageUtilsTest.java | 4 +- .../basemap/util/VerifyMonacoTest.java | 62 + flatmap-benchmarks/pom.xml | 31 +- ...pTilesMapping.java => BasemapMapping.java} | 6 +- flatmap-core/pom.xml | 45 +- .../com/onthegomap/flatmap/FlatmapRunner.java | 31 +- .../onthegomap/flatmap/ForwardingProfile.java | 2 +- .../java/com/onthegomap/flatmap/Profile.java | 23 +- .../com/onthegomap/flatmap/VectorTile.java | 180 +- .../flatmap/collection/AppendStoreMmap.java | 31 + .../flatmap/collection/ExternalMergeSort.java | 28 +- .../flatmap/collection/FeatureGroup.java | 38 +- .../onthegomap/flatmap/config/Arguments.java | 14 +- .../flatmap/config/FlatmapConfig.java | 6 + .../flatmap/config/MbtilesMetadata.java | 2 +- .../flatmap/expression/Expression.java | 2 +- .../onthegomap/flatmap/geo/GeometryType.java | 16 +- .../com/onthegomap/flatmap/geo/TileCoord.java | 15 +- .../flatmap/{mbiles => mbtiles}/Mbtiles.java | 5 +- .../{mbiles => mbtiles}/MbtilesWriter.java | 43 +- .../onthegomap/flatmap/mbtiles/Verify.java | 236 + .../flatmap/reader/osm/OsmInputFile.java | 2 +- .../flatmap/reader/osm/OsmReader.java | 18 +- .../flatmap/render/FeatureRenderer.java | 17 +- .../flatmap/stats/ProgressLoggers.java | 38 +- .../flatmap/stats/PrometheusStats.java | 4 +- .../com/onthegomap/flatmap/stats/Stats.java | 32 +- .../com/onthegomap/flatmap/stats/Timers.java | 3 +- .../com/onthegomap/flatmap/util/AwsOsm.java | 89 + .../onthegomap/flatmap/util/Downloader.java | 213 +- .../com/onthegomap/flatmap/util/Format.java | 10 + .../onthegomap/flatmap/util/Geofabrik.java | 12 +- .../com/onthegomap/flatmap/util/Gzip.java | 24 + .../onthegomap/flatmap/util/LayerStats.java | 4 +- .../com/onthegomap/flatmap/util/Wikidata.java | 11 +- .../onthegomap/flatmap/util/ZoomFunction.java | 4 +- .../com/onthegomap/flatmap/worker/Worker.java | 4 +- .../assemblies/{fatjar.xml => with-deps.xml} | 2 +- .../onthegomap/flatmap/FeatureMergeTest.java | 5 +- .../com/onthegomap/flatmap/FlatmapTests.java | 57 +- .../com/onthegomap/flatmap/TestUtils.java | 51 +- .../onthegomap/flatmap/VectorTileTest.java | 80 +- .../flatmap/collection/FeatureGroupTest.java | 15 + .../flatmap/config/ArgumentsTest.java | 11 + .../{mbiles => mbtiles}/MbtilesTest.java | 2 +- .../flatmap/mbtiles/VerifyTest.java | 91 + .../flatmap/reader/osm/OsmReaderTest.java | 31 +- .../flatmap/stats/ProgressLoggersTest.java | 22 +- .../flatmap/stats/PrometheusStatsTest.java | 2 +- .../onthegomap/flatmap/util/AwsOsmTest.java | 188 + .../flatmap/util/DownloaderTest.java | 112 + .../com/onthegomap/flatmap/util/GzipTest.java | 21 + .../flatmap/util/LayerStatsTest.java | 2 +- flatmap-dist/README.md | 3 + flatmap-dist/pom.xml | 125 + .../java/com/onthegomap/flatmap/Main.java | 58 + flatmap-examples/README.md | 190 + flatmap-examples/child.pom.xml | 50 + flatmap-examples/pom.xml | 91 +- .../flatmap/examples/BikeRouteOverlay.java | 11 +- .../flatmap/examples/ToiletsOverlay.java | 6 +- .../examples/ToiletsOverlayLowLevelApi.java | 8 +- .../examples/BikeRouteOverlayTest.java | 2 +- .../ToiletsOverlayLowLevelApiTest.java | 2 +- .../flatmap/examples/ToiletsProfileTest.java | 2 +- flatmap-openmaptiles/pom.xml | 83 - mvnw | 310 ++ mvnw.cmd | 182 + planet-logs/logs-daylight.txt | 4363 ++++++++++++++++ planet-logs/planet-logs-c5ad.txt | 1382 +++++ planet-logs/planet-logs.txt | 4582 +++++++++++++++++ planet-logs/planet.nps | Bin 0 -> 2131334 bytes pom.xml | 170 +- quickstart-from-source.sh | 12 + quickstart-openmaptiles.sh | 16 - scripts/build-release.sh | 5 + scripts/build.sh | 4 +- scripts/check-doc-links.sh | 7 + scripts/check-mbtiles.sh | 7 + scripts/check-monaco.sh | 7 + scripts/download-osm.sh | 23 - scripts/download-other-sources.sh | 14 - scripts/push-release.sh | 16 + scripts/regenerate-openmaptiles.sh | 6 +- ...ate-protobuf.sh => regenerate-protobuf.sh} | 0 scripts/set-versions.sh | 13 + scripts/test-release.sh | 37 + scripts/ubuntu_vm_setup.sh | 24 - 160 files changed, 15054 insertions(+), 744 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/cache-sources-action/action.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/docs_mlc_config.json create mode 100644 .github/workflows/performance.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/snapshot.yml create mode 100644 .github/workflows/update-pr.yml create mode 100644 .mvn/wrapper/MavenWrapperDownloader.java create mode 100644 .mvn/wrapper/maven-wrapper.jar create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100644 ARCHITECTURE.md create mode 100644 CONTRIBUTING.md create mode 100644 PLANET.md create mode 100644 README.md create mode 100644 diagrams/architecture.png create mode 100644 diagrams/architecture.puml create mode 100644 diagrams/demo.png create mode 100644 diagrams/slicing.png create mode 100644 flatmap-basemap/README.md create mode 100644 flatmap-basemap/pom.xml rename flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/OpenMapTilesMain.java => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap/BasemapMain.java (65%) rename flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles/OpenMapTilesProfile.java => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap/BasemapProfile.java (94%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/Generate.java (94%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/Layer.java (71%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/generated/OpenMapTilesSchema.java (97%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/generated/Tables.java (99%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/AerodromeLabel.java (89%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Aeroway.java (94%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Boundary.java (97%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Building.java (94%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Housenumber.java (93%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Landcover.java (95%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Landuse.java (90%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/MountainPeak.java (91%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Park.java (92%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Place.java (96%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Poi.java (91%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Transportation.java (95%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/TransportationName.java (93%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Water.java (92%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/WaterName.java (94%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/layers/Waterway.java (91%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/util/LanguageUtils.java (97%) rename {flatmap-openmaptiles/src/main/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap}/util/Utils.java (97%) create mode 100644 flatmap-basemap/src/main/java/com/onthegomap/flatmap/basemap/util/VerifyMonaco.java rename flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles/OpenMapTilesProfileTest.java => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap/BasemapProfileTest.java (86%) rename flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles/OpenMapTilesTest.java => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap/BasemapTest.java (86%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/GenerateTest.java (98%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/AbstractLayerTest.java (92%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/AerodromeLabelTest.java (98%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/AerowayTest.java (97%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/BoundaryTest.java (98%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/BuildingTest.java (97%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/HousenumberTest.java (92%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/LandcoverTest.java (97%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/LanduseTest.java (93%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/MountainPeakTest.java (99%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/ParkTest.java (98%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/PlaceTest.java (97%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/PoiTest.java (98%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/TransportationTest.java (98%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/WaterNameTest.java (92%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/WaterTest.java (93%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/layers/WaterwayTest.java (96%) rename {flatmap-openmaptiles/src/test/java/com/onthegomap/flatmap/openmaptiles => flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap}/util/LanguageUtilsTest.java (97%) create mode 100644 flatmap-basemap/src/test/java/com/onthegomap/flatmap/basemap/util/VerifyMonacoTest.java rename flatmap-benchmarks/src/main/java/com/onthegomap/flatmap/benchmarks/{OpenMapTilesMapping.java => BasemapMapping.java} (93%) rename flatmap-core/src/main/java/com/onthegomap/flatmap/{mbiles => mbtiles}/Mbtiles.java (99%) rename flatmap-core/src/main/java/com/onthegomap/flatmap/{mbiles => mbtiles}/MbtilesWriter.java (92%) create mode 100644 flatmap-core/src/main/java/com/onthegomap/flatmap/mbtiles/Verify.java create mode 100644 flatmap-core/src/main/java/com/onthegomap/flatmap/util/AwsOsm.java create mode 100644 flatmap-core/src/main/java/com/onthegomap/flatmap/util/Gzip.java rename flatmap-core/src/main/resources/assemblies/{fatjar.xml => with-deps.xml} (98%) rename flatmap-core/src/test/java/com/onthegomap/flatmap/{mbiles => mbtiles}/MbtilesTest.java (99%) create mode 100644 flatmap-core/src/test/java/com/onthegomap/flatmap/mbtiles/VerifyTest.java create mode 100644 flatmap-core/src/test/java/com/onthegomap/flatmap/util/AwsOsmTest.java create mode 100644 flatmap-core/src/test/java/com/onthegomap/flatmap/util/DownloaderTest.java create mode 100644 flatmap-core/src/test/java/com/onthegomap/flatmap/util/GzipTest.java create mode 100644 flatmap-dist/README.md create mode 100644 flatmap-dist/pom.xml create mode 100644 flatmap-dist/src/main/java/com/onthegomap/flatmap/Main.java create mode 100644 flatmap-examples/README.md create mode 100644 flatmap-examples/child.pom.xml delete mode 100644 flatmap-openmaptiles/pom.xml create mode 100755 mvnw create mode 100644 mvnw.cmd create mode 100644 planet-logs/logs-daylight.txt create mode 100644 planet-logs/planet-logs-c5ad.txt create mode 100644 planet-logs/planet-logs.txt create mode 100644 planet-logs/planet.nps create mode 100755 quickstart-from-source.sh delete mode 100755 quickstart-openmaptiles.sh create mode 100755 scripts/build-release.sh create mode 100755 scripts/check-doc-links.sh create mode 100755 scripts/check-mbtiles.sh create mode 100755 scripts/check-monaco.sh delete mode 100755 scripts/download-osm.sh delete mode 100755 scripts/download-other-sources.sh create mode 100755 scripts/push-release.sh rename scripts/{generate-protobuf.sh => regenerate-protobuf.sh} (100%) create mode 100755 scripts/set-versions.sh create mode 100755 scripts/test-release.sh delete mode 100755 scripts/ubuntu_vm_setup.sh diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..b15f5a89 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help improve Flatmap +title: "[BUG] " +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Download data from '...' +2. Run command '....' +3. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem (include tile IDs or latitude/longitude for visual issues with generated maps) + +**Environment (please complete the following information):** + - Hardware: [e.g. 2015 Macbook Pro] + - OS: [e.g. MacOS 10.15.7] + - Java version and distribution: [e.g. Eclipse Temurin 17.35] + - Maven version: [e.g. 3.8.1] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..11968690 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for Flatmap +title: "[FEATURE] " +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/cache-sources-action/action.yml b/.github/cache-sources-action/action.yml new file mode 100644 index 00000000..c464ca6f --- /dev/null +++ b/.github/cache-sources-action/action.yml @@ -0,0 +1,22 @@ +# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions +name: 'Cache data/sources' +description: 'Save/restore data/sources cache' +inputs: + basedir: + description: 'Base dir for computing file hash' + required: false + default: '' +runs: + using: 'composite' + steps: + - name: Get Date + id: get-data + run: | + echo "::set-output name=hash::${{ hashFiles('**/FlatmapRunner.java', '**/Downloader.java', '**/Geofabrik.java', '**/BasemapMain.java') }}" + echo "::set-output name=date::$(date -u "+%Y-%m-%d")" + shell: bash + working-directory: ${{ inputs.basedir }} + - uses: actions/cache@v2 + with: + path: data/sources + key: data-sources-${{ steps.get-data.outputs.date }}-${{ steps.get-data.outputs.hash }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..4408eec3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,33 @@ +version: 2 +updates: + - package-ecosystem: maven + directory: "/" + open-pull-requests-limit: 1 + schedule: + interval: daily + time: "04:30" + timezone: America/New_York + labels: + - dependencies + ignore: + - dependency-name: "com.graphhopper:graphhopper-reader-osm" + - package-ecosystem: maven + directory: "/flatmap-examples" + open-pull-requests-limit: 1 + schedule: + interval: daily + time: "04:30" + timezone: America/New_York + labels: + - dependencies + ignore: + - dependency-name: "com.onthegomap.flatmap:*" + - package-ecosystem: github-actions + directory: "/" + open-pull-requests-limit: 1 + schedule: + interval: daily + time: "04:30" + timezone: America/New_York + labels: + - dependencies diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..716b6e5f --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,21 @@ +--- +name: Docs +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + markdown-link-check: + name: Broken Links + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run link check + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'no' + use-verbose-mode: 'yes' + config-file: '.github/workflows/docs_mlc_config.json' diff --git a/.github/workflows/docs_mlc_config.json b/.github/workflows/docs_mlc_config.json new file mode 100644 index 00000000..c5c444af --- /dev/null +++ b/.github/workflows/docs_mlc_config.json @@ -0,0 +1,13 @@ +{ + "ignorePatterns": [ + { + "pattern": "^https://github.com/onthegomap/flatmap/.*$" + }, + { + "pattern": "^https://onthegomap.github.io/.*$" + }, + { + "pattern": "^http://localhost.*$" + } + ] +} diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d068c7c1..622ae721 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,7 +1,7 @@ # This workflow will build a Java project with Maven # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven -name: Java CI with Maven +name: CI on: push: @@ -10,23 +10,94 @@ on: branches: [ main ] jobs: + # TODO: add format/checkstyle build: - - runs-on: ubuntu-latest - timeout-minutes: 10 - + name: Java ${{ matrix.jdk }} / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + # os: [ ubuntu-latest, macos-latest, windows-latest ] + jdk: [ 17 ] + # include: + # - os: ubuntu-latest + # jdk: 16 + runs-on: ${{ matrix.os }} + timeout-minutes: 5 steps: - uses: actions/checkout@v2 - - name: Set up JDK 16 + - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v2 with: - java-version: '16' - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v2 + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' + - name: Build with mvnw (linux/mac) + if: ${{ !contains(matrix.os, 'windows') }} + run: ./mvnw --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml + - name: Build with mvnw.cmd (windows) + if: ${{ contains(matrix.os, 'windows') }} + run: mvnw.cmd --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml + shell: cmd + + examples: + name: Example project + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 17 + uses: actions/setup-java@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Build with Maven - run: mvn -B package --file pom.xml + java-version: 17 + distribution: 'temurin' + # do not cache to ensure we can resolve dependencies + server-id: 'github-flatmap' + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Build and test + run: mvn --batch-mode -no-transfer-progress package --file pom.xml + working-directory: flatmap-examples + # workaround for github maven packages not supporting anonymous access (https://github.community/t/download-from-github-package-registry-without-authentication/14407/111) + env: + MAVEN_USERNAME: 'flatmapbot' + MAVEN_PASSWORD: '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' + - name: Find jar + run: mv target/*with-deps.jar ./run.jar + working-directory: flatmap-examples + - name: Run + run: java -jar run.jar --osm-path=../flatmap-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles + working-directory: flatmap-examples + - name: Verify + run: java -cp run.jar com.onthegomap.flatmap.mbtiles.Verify data/out.mbtiles + working-directory: flatmap-examples + + run: + name: Build / Run + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v2 + - name: Cache data/sources + uses: ./.github/cache-sources-action + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + + - name: Build this branch + run: ./mvnw -DskipTests -Dimage.version=CI_ONLY --batch-mode -no-transfer-progress package jib:dockerBuild --file pom.xml + + - name: Download data (java) + run: java -jar flatmap-dist/target/*with-deps.jar --only-download --area=monaco + + - name: Download wikidata (java) + run: java -jar flatmap-dist/target/*with-deps.jar --only-fetch-wikidata --area=monaco + + - name: Verify build + run: ./scripts/test-release.sh CI_ONLY + env: + SKIP_EXAMPLE_PROJECT: true diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 00000000..6750c335 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,92 @@ +# This workflow builds a map using the base and branch commit of a PR and uploads +# the logs as an artifact that update-pr.yml uses to add back as a comment. + +name: Performance + +on: + pull_request: + branches: [ main ] + +env: + # For performance tests, run this branch against main with: + AREA: rhode island + RAM: 4g + # Also pick up a good chunk of the atlantic ocean to catch any regressions in repeated tile performance + # Omit to infer from .osm.pbf bounds + BOUNDS_ARG: "--bounds=-74.07,21.34,-17.84,43.55" + +jobs: + performance: + name: Performance Test + runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: true + steps: + - name: 'Cancel previous runs' + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + - name: 'Checkout branch' + uses: actions/checkout@v2 + with: + path: branch + - name: 'Checkout base' + uses: actions/checkout@v2 + with: + path: base + ref: ${{ github.event.pull_request.base.sha }} + - name: Cache data/sources + uses: ./branch/.github/cache-sources-action + with: + basedir: branch + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - uses: actions/setup-node@v2 + with: + node-version: '14' + - run: npm install -g strip-ansi-cli@3.0.2 + + - name: 'Build branch' + run: ./scripts/build.sh + working-directory: branch + - name: 'Build base' + run: ./scripts/build.sh + working-directory: base + + - name: 'Download data' + run: | + set -eo pipefail + cp base/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + cp branch/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + + - name: 'Store build info' + run: | + mkdir build-info + echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha + echo "${{ github.sha }}" > build-info/branch_sha + echo "${{ github.event.number }}" > build-info/pull_request_number + + - name: 'Run branch' + run: | + rm -f data/out.mbtiles + cp branch/flatmap-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/branchlogs.txt + - name: 'Run base' + run: | + rm -f data/out.mbtiles + cp base/flatmap-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/baselogs.txt + + - name: 'Upload build-info' + uses: actions/upload-artifact@v2 + with: + name: build-info + path: ./build-info diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..4464626b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: Publish a Release +on: + workflow_dispatch: + inputs: + version: + description: 'Version without leading "v" (1.0, 2.3.4, 0.2-alpha, 5.9-beta.3)' + required: true + default: '' + image_tags: + description: 'Extra docker image tags ("latest,test")' + required: true + default: 'latest' +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + packages: write + steps: + - name: Ensure version does not start with 'v' + uses: actions/github-script@v5 + with: + github-token: ${{ github.token }} + script: | + version = context.payload.inputs.version; + if (/^v/.test(version)) throw new Error("Bad version number: " + version) + - uses: actions/checkout@v2 + - name: Cache data/sources + uses: ./.github/cache-sources-action + - uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Check tag does not exist yet + run: if git rev-list "v${{ github.event.inputs.version }}"; then echo "Tag already exists. Aborting the release process."; exit 1; fi + + - run: ./scripts/set-versions.sh "${{ github.event.inputs.version }}" + - run: ./scripts/build-release.sh + - run: ./scripts/test-release.sh "${{ github.event.inputs.version }}" + - name: Create tag + uses: actions/github-script@v5 + with: + github-token: ${{ github.token }} + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: "refs/tags/v${{ github.event.inputs.version }}", + sha: context.sha + }) + - run: mv flatmap-dist/target/*with-deps.jar flatmap.jar + - name: Log in to the Container Registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + fail_on_unmatched_files: true + tag_name: v${{ github.event.inputs.version }} + draft: true + files: | + flatmap.jar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./scripts/push-release.sh ${{ github.event.inputs.version }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAGS: ${{ github.event.inputs.image_tags }} diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml new file mode 100644 index 00000000..6ee21b82 --- /dev/null +++ b/.github/workflows/snapshot.yml @@ -0,0 +1,42 @@ +# This workflow builds a map using the base and branch commit of a PR and uploads +# the logs as an artifact that update-pr.yml uses to add back as a comment. + +name: Publish a Snapshot + +on: + workflow_dispatch: + inputs: + image_tags: + description: 'Extra docker image tags ("latest,test")' + required: true + default: 'latest,snapshot' + +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v2 + - name: Cache data/sources + uses: ./.github/cache-sources-action + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - run: ./scripts/build-release.sh + - run: ./scripts/test-release.sh + - name: Log in to the Container Registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: ./scripts/push-release.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAGS: ${{ github.event.inputs.image_tags }} diff --git a/.github/workflows/update-pr.yml b/.github/workflows/update-pr.yml new file mode 100644 index 00000000..5d143c2a --- /dev/null +++ b/.github/workflows/update-pr.yml @@ -0,0 +1,89 @@ +# This workflow posts the result of a performance test back to the pull request as a comment. +# Needs to be separate from CI because it has elevated privileges so only runs from main branch. + +name: Update PR + +on: + workflow_run: + workflows: [ "Performance" ] + types: + - completed + +jobs: + updatepr: + runs-on: ubuntu-latest + continue-on-error: true + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + timeout-minutes: 5 + steps: + # report status back to pull request + - uses: haya14busa/action-workflow_run-status@v1 + - uses: actions/checkout@v2 + - name: 'Download branch build info' + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + name: build-info + path: build-info + repo: onthegomap/flatmap + - name: 'Get build info' + id: build_info + run: echo "::set-output name=pr_number::$(cat build-info/pull_request_number)" + - name: 'Build comment-body' + run: | + cat build-info/branchlogs.txt | sed -n '/^.*Tile stats/,$p' > branchsummary.txt + cat build-info/branchlogs.txt | sed -n '/^.*Exception in thread/,$p' >> branchsummary.txt + cat build-info/baselogs.txt | sed -n '/^.*Tile stats:/,$p' > basesummary.txt + cat build-info/baselogs.txt | sed -n '/^.*Exception in thread/,$p' >> basesummary.txt + + cat << EOF > comment-body.txt + + + + + + + + + + + +
Base $(cat build-info/base_sha)This Branch $(cat build-info/branch_sha)
+ + \`\`\` + $(cat basesummary.txt) + \`\`\` + + + \`\`\` + $(cat build-info/branchlogs.txt | sed -n '/^.*Tile stats:/,$p') + \`\`\` +
+ + https://github.com/onthegomap/flatmap/actions/runs/${{ github.event.workflow_run.id }} + +
ℹ️ Base Logs $(cat build-info/base_sha) + + \`\`\` + $(cat build-info/baselogs.txt) + \`\`\` +
+ +
ℹ️ This Branch Logs $(cat build-info/branch_sha) + + \`\`\` + $(cat build-info/branchlogs.txt) + \`\`\` +
+ EOF + + - name: 'Dump comment body' + run: cat comment-body.txt + + - uses: marocchino/sticky-pull-request-comment@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + path: comment-body.txt + header: performance-tests + number: ${{ steps.build_info.outputs.pr_number }} diff --git a/.gitignore b/.gitignore index 04faab1b..242d712d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,10 @@ target/ *.jar +!.mvn/wrapper/*.jar *.log +*/.idea .idea/* *.iml !.idea/codeStyles diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 48d9b65e..b92b5ecb 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -105,6 +105,9 @@ + + @@ -204,6 +207,9 @@