diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..b15f5a89 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help improve Flatmap +title: "[BUG] " +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Download data from '...' +2. Run command '....' +3. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem (include tile IDs or latitude/longitude for visual issues with generated maps) + +**Environment (please complete the following information):** + - Hardware: [e.g. 2015 Macbook Pro] + - OS: [e.g. MacOS 10.15.7] + - Java version and distribution: [e.g. Eclipse Temurin 17.35] + - Maven version: [e.g. 3.8.1] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..11968690 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for Flatmap +title: "[FEATURE] " +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/cache-sources-action/action.yml b/.github/cache-sources-action/action.yml new file mode 100644 index 00000000..c464ca6f --- /dev/null +++ b/.github/cache-sources-action/action.yml @@ -0,0 +1,22 @@ +# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/metadata-syntax-for-github-actions +name: 'Cache data/sources' +description: 'Save/restore data/sources cache' +inputs: + basedir: + description: 'Base dir for computing file hash' + required: false + default: '' +runs: + using: 'composite' + steps: + - name: Get Date + id: get-data + run: | + echo "::set-output name=hash::${{ hashFiles('**/FlatmapRunner.java', '**/Downloader.java', '**/Geofabrik.java', '**/BasemapMain.java') }}" + echo "::set-output name=date::$(date -u "+%Y-%m-%d")" + shell: bash + working-directory: ${{ inputs.basedir }} + - uses: actions/cache@v2 + with: + path: data/sources + key: data-sources-${{ steps.get-data.outputs.date }}-${{ steps.get-data.outputs.hash }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..4408eec3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,33 @@ +version: 2 +updates: + - package-ecosystem: maven + directory: "/" + open-pull-requests-limit: 1 + schedule: + interval: daily + time: "04:30" + timezone: America/New_York + labels: + - dependencies + ignore: + - dependency-name: "com.graphhopper:graphhopper-reader-osm" + - package-ecosystem: maven + directory: "/flatmap-examples" + open-pull-requests-limit: 1 + schedule: + interval: daily + time: "04:30" + timezone: America/New_York + labels: + - dependencies + ignore: + - dependency-name: "com.onthegomap.flatmap:*" + - package-ecosystem: github-actions + directory: "/" + open-pull-requests-limit: 1 + schedule: + interval: daily + time: "04:30" + timezone: America/New_York + labels: + - dependencies diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..716b6e5f --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,21 @@ +--- +name: Docs +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + markdown-link-check: + name: Broken Links + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run link check + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'no' + use-verbose-mode: 'yes' + config-file: '.github/workflows/docs_mlc_config.json' diff --git a/.github/workflows/docs_mlc_config.json b/.github/workflows/docs_mlc_config.json new file mode 100644 index 00000000..c5c444af --- /dev/null +++ b/.github/workflows/docs_mlc_config.json @@ -0,0 +1,13 @@ +{ + "ignorePatterns": [ + { + "pattern": "^https://github.com/onthegomap/flatmap/.*$" + }, + { + "pattern": "^https://onthegomap.github.io/.*$" + }, + { + "pattern": "^http://localhost.*$" + } + ] +} diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d068c7c1..622ae721 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,7 +1,7 @@ # This workflow will build a Java project with Maven # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven -name: Java CI with Maven +name: CI on: push: @@ -10,23 +10,94 @@ on: branches: [ main ] jobs: + # TODO: add format/checkstyle build: - - runs-on: ubuntu-latest - timeout-minutes: 10 - + name: Java ${{ matrix.jdk }} / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + # os: [ ubuntu-latest, macos-latest, windows-latest ] + jdk: [ 17 ] + # include: + # - os: ubuntu-latest + # jdk: 16 + runs-on: ${{ matrix.os }} + timeout-minutes: 5 steps: - uses: actions/checkout@v2 - - name: Set up JDK 16 + - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v2 with: - java-version: '16' - distribution: 'adopt' - - name: Cache Maven packages - uses: actions/cache@v2 + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' + - name: Build with mvnw (linux/mac) + if: ${{ !contains(matrix.os, 'windows') }} + run: ./mvnw --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml + - name: Build with mvnw.cmd (windows) + if: ${{ contains(matrix.os, 'windows') }} + run: mvnw.cmd --batch-mode -no-transfer-progress package jib:buildTar --file pom.xml + shell: cmd + + examples: + name: Example project + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 17 + uses: actions/setup-java@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Build with Maven - run: mvn -B package --file pom.xml + java-version: 17 + distribution: 'temurin' + # do not cache to ensure we can resolve dependencies + server-id: 'github-flatmap' + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Build and test + run: mvn --batch-mode -no-transfer-progress package --file pom.xml + working-directory: flatmap-examples + # workaround for github maven packages not supporting anonymous access (https://github.community/t/download-from-github-package-registry-without-authentication/14407/111) + env: + MAVEN_USERNAME: 'flatmapbot' + MAVEN_PASSWORD: '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' + - name: Find jar + run: mv target/*with-deps.jar ./run.jar + working-directory: flatmap-examples + - name: Run + run: java -jar run.jar --osm-path=../flatmap-core/src/test/resources/monaco-latest.osm.pbf --mbtiles=data/out.mbtiles + working-directory: flatmap-examples + - name: Verify + run: java -cp run.jar com.onthegomap.flatmap.mbtiles.Verify data/out.mbtiles + working-directory: flatmap-examples + + run: + name: Build / Run + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v2 + - name: Cache data/sources + uses: ./.github/cache-sources-action + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + + - name: Build this branch + run: ./mvnw -DskipTests -Dimage.version=CI_ONLY --batch-mode -no-transfer-progress package jib:dockerBuild --file pom.xml + + - name: Download data (java) + run: java -jar flatmap-dist/target/*with-deps.jar --only-download --area=monaco + + - name: Download wikidata (java) + run: java -jar flatmap-dist/target/*with-deps.jar --only-fetch-wikidata --area=monaco + + - name: Verify build + run: ./scripts/test-release.sh CI_ONLY + env: + SKIP_EXAMPLE_PROJECT: true diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 00000000..6750c335 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,92 @@ +# This workflow builds a map using the base and branch commit of a PR and uploads +# the logs as an artifact that update-pr.yml uses to add back as a comment. + +name: Performance + +on: + pull_request: + branches: [ main ] + +env: + # For performance tests, run this branch against main with: + AREA: rhode island + RAM: 4g + # Also pick up a good chunk of the atlantic ocean to catch any regressions in repeated tile performance + # Omit to infer from .osm.pbf bounds + BOUNDS_ARG: "--bounds=-74.07,21.34,-17.84,43.55" + +jobs: + performance: + name: Performance Test + runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: true + steps: + - name: 'Cancel previous runs' + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + - name: 'Checkout branch' + uses: actions/checkout@v2 + with: + path: branch + - name: 'Checkout base' + uses: actions/checkout@v2 + with: + path: base + ref: ${{ github.event.pull_request.base.sha }} + - name: Cache data/sources + uses: ./branch/.github/cache-sources-action + with: + basedir: branch + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - uses: actions/setup-node@v2 + with: + node-version: '14' + - run: npm install -g strip-ansi-cli@3.0.2 + + - name: 'Build branch' + run: ./scripts/build.sh + working-directory: branch + - name: 'Build base' + run: ./scripts/build.sh + working-directory: base + + - name: 'Download data' + run: | + set -eo pipefail + cp base/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + cp branch/flatmap-dist/target/*with-deps.jar run.jar && java -jar run.jar --only-download --area="${{ env.AREA }}" + + - name: 'Store build info' + run: | + mkdir build-info + echo "${{ github.event.pull_request.base.sha }}" > build-info/base_sha + echo "${{ github.sha }}" > build-info/branch_sha + echo "${{ github.event.number }}" > build-info/pull_request_number + + - name: 'Run branch' + run: | + rm -f data/out.mbtiles + cp branch/flatmap-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/branchlogs.txt + - name: 'Run base' + run: | + rm -f data/out.mbtiles + cp base/flatmap-dist/target/*with-deps.jar run.jar + java -Xms${{ env.RAM }} -Xmx${{ env.RAM }} -jar run.jar --area="${{ env.AREA }}" "${{ env.BOUNDS_ARG }}" --mbtiles=data/out.mbtiles 2>&1 | tee log + ls -alh run.jar | tee -a log + cat log | strip-ansi > build-info/baselogs.txt + + - name: 'Upload build-info' + uses: actions/upload-artifact@v2 + with: + name: build-info + path: ./build-info diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..4464626b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: Publish a Release +on: + workflow_dispatch: + inputs: + version: + description: 'Version without leading "v" (1.0, 2.3.4, 0.2-alpha, 5.9-beta.3)' + required: true + default: '' + image_tags: + description: 'Extra docker image tags ("latest,test")' + required: true + default: 'latest' +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + packages: write + steps: + - name: Ensure version does not start with 'v' + uses: actions/github-script@v5 + with: + github-token: ${{ github.token }} + script: | + version = context.payload.inputs.version; + if (/^v/.test(version)) throw new Error("Bad version number: " + version) + - uses: actions/checkout@v2 + - name: Cache data/sources + uses: ./.github/cache-sources-action + - uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Check tag does not exist yet + run: if git rev-list "v${{ github.event.inputs.version }}"; then echo "Tag already exists. Aborting the release process."; exit 1; fi + + - run: ./scripts/set-versions.sh "${{ github.event.inputs.version }}" + - run: ./scripts/build-release.sh + - run: ./scripts/test-release.sh "${{ github.event.inputs.version }}" + - name: Create tag + uses: actions/github-script@v5 + with: + github-token: ${{ github.token }} + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: "refs/tags/v${{ github.event.inputs.version }}", + sha: context.sha + }) + - run: mv flatmap-dist/target/*with-deps.jar flatmap.jar + - name: Log in to the Container Registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + fail_on_unmatched_files: true + tag_name: v${{ github.event.inputs.version }} + draft: true + files: | + flatmap.jar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: ./scripts/push-release.sh ${{ github.event.inputs.version }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAGS: ${{ github.event.inputs.image_tags }} diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml new file mode 100644 index 00000000..6ee21b82 --- /dev/null +++ b/.github/workflows/snapshot.yml @@ -0,0 +1,42 @@ +# This workflow builds a map using the base and branch commit of a PR and uploads +# the logs as an artifact that update-pr.yml uses to add back as a comment. + +name: Publish a Snapshot + +on: + workflow_dispatch: + inputs: + image_tags: + description: 'Extra docker image tags ("latest,test")' + required: true + default: 'latest,snapshot' + +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v2 + - name: Cache data/sources + uses: ./.github/cache-sources-action + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + - run: ./scripts/build-release.sh + - run: ./scripts/test-release.sh + - name: Log in to the Container Registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - run: ./scripts/push-release.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IMAGE_TAGS: ${{ github.event.inputs.image_tags }} diff --git a/.github/workflows/update-pr.yml b/.github/workflows/update-pr.yml new file mode 100644 index 00000000..5d143c2a --- /dev/null +++ b/.github/workflows/update-pr.yml @@ -0,0 +1,89 @@ +# This workflow posts the result of a performance test back to the pull request as a comment. +# Needs to be separate from CI because it has elevated privileges so only runs from main branch. + +name: Update PR + +on: + workflow_run: + workflows: [ "Performance" ] + types: + - completed + +jobs: + updatepr: + runs-on: ubuntu-latest + continue-on-error: true + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + timeout-minutes: 5 + steps: + # report status back to pull request + - uses: haya14busa/action-workflow_run-status@v1 + - uses: actions/checkout@v2 + - name: 'Download branch build info' + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + name: build-info + path: build-info + repo: onthegomap/flatmap + - name: 'Get build info' + id: build_info + run: echo "::set-output name=pr_number::$(cat build-info/pull_request_number)" + - name: 'Build comment-body' + run: | + cat build-info/branchlogs.txt | sed -n '/^.*Tile stats/,$p' > branchsummary.txt + cat build-info/branchlogs.txt | sed -n '/^.*Exception in thread/,$p' >> branchsummary.txt + cat build-info/baselogs.txt | sed -n '/^.*Tile stats:/,$p' > basesummary.txt + cat build-info/baselogs.txt | sed -n '/^.*Exception in thread/,$p' >> basesummary.txt + + cat << EOF > comment-body.txt + + + + + + + + + + + +
Base $(cat build-info/base_sha)This Branch $(cat build-info/branch_sha)
+ + \`\`\` + $(cat basesummary.txt) + \`\`\` + + + \`\`\` + $(cat build-info/branchlogs.txt | sed -n '/^.*Tile stats:/,$p') + \`\`\` +
+ + https://github.com/onthegomap/flatmap/actions/runs/${{ github.event.workflow_run.id }} + +
ℹ️ Base Logs $(cat build-info/base_sha) + + \`\`\` + $(cat build-info/baselogs.txt) + \`\`\` +
+ +
ℹ️ This Branch Logs $(cat build-info/branch_sha) + + \`\`\` + $(cat build-info/branchlogs.txt) + \`\`\` +
+ EOF + + - name: 'Dump comment body' + run: cat comment-body.txt + + - uses: marocchino/sticky-pull-request-comment@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + path: comment-body.txt + header: performance-tests + number: ${{ steps.build_info.outputs.pr_number }} diff --git a/.gitignore b/.gitignore index 04faab1b..242d712d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,10 @@ target/ *.jar +!.mvn/wrapper/*.jar *.log +*/.idea .idea/* *.iml !.idea/codeStyles diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 48d9b65e..b92b5ecb 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -105,6 +105,9 @@ + + @@ -204,6 +207,9 @@