From 09100d77f46796579cefa38b6fb1df79188e6089 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 12:07:53 -0400 Subject: [PATCH 01/18] Update snapcraft.yaml Update package name to match snapcraft listing --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 6e496d43..8e290ba5 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,4 +1,4 @@ -name: opendronemap +name: opendronemap-core18-test adopt-info: odm grade: devel confinement: strict From 1f470239475981aae0aeb2527c08ba97fd2167cd Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:28:28 -0400 Subject: [PATCH 02/18] Create snapcraftbuild.yml --- .github/workflows/snapcraftbuild.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/snapcraftbuild.yml diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml new file mode 100644 index 00000000..4455246a --- /dev/null +++ b/.github/workflows/snapcraftbuild.yml @@ -0,0 +1,11 @@ +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: snapcore/action-build@v1 + id: snapcraft + - uses: actions/upload-artifact@v2 + with: + name: snap + path: ${{ steps.snapcraft.outputs.snap }} From 63141dbae469a3978248cd2e671f57cefc80d15e Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:30:16 -0400 Subject: [PATCH 03/18] Update snapcraftbuild.yml --- .github/workflows/snapcraftbuild.yml | 51 ++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index 4455246a..c0b5eb82 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -1,11 +1,40 @@ -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: snapcore/action-build@v1 - id: snapcraft - - uses: actions/upload-artifact@v2 - with: - name: snap - path: ${{ steps.snapcraft.outputs.snap }} +name: 'Snapcraft Build' +description: 'Build a Snapcraft project' +author: 'James Henstridge' +branding: + icon: 'package' + color: 'orange' +inputs: + path: + description: 'The location of the Snapcraft project. Defaults to the base of the repository' + default: '.' + build-info: + description: > + Whether to include build information in the resulting snap. + This will add snap/manifest.yaml and snap/snapcraft.yaml files + to the snap. The Snap Store can use this information to detect + snaps with security vulnerabilities introduced through + stage-packages. + Proprietary applications may want to disable this due to + the information leakage. + default: 'true' + snapcraft-channel: + description: > + The Snapcraft channel to use + By default, the action will use the stable version of Snapcraft + to build the project. This parameter can be used to instead + select a different channel such as beta, candidate, or edge. + default: 'stable' + snapcraft-args: + description: > + Additional arguments to pass to Snapcraft + Some experimental Snapcraft features are disabled by default and + must be turned on via a `--enable-experimental-*` command line + argument. This parameter can be used to turn on such features. + default: '' +outputs: + snap: + description: 'The file name of the resulting snap.' +runs: + using: 'node12' + main: 'dist/index.js' From ec81cf6a07c3b4abe021cfb465a5bcfd18fddcb1 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:46:21 -0400 Subject: [PATCH 04/18] Update snapcraftbuild.yml --- .github/workflows/snapcraftbuild.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index c0b5eb82..4d2914ee 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -1,3 +1,5 @@ +on: + workflow_dispatch: name: 'Snapcraft Build' description: 'Build a Snapcraft project' author: 'James Henstridge' From 138cb53cb7b6c25c5c6674cd5c7d081d69e65168 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:48:35 -0400 Subject: [PATCH 05/18] Update snapcraftbuild.yml --- .github/workflows/snapcraftbuild.yml | 52 +++++++--------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index 4d2914ee..cab0660b 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -1,42 +1,14 @@ on: workflow_dispatch: -name: 'Snapcraft Build' -description: 'Build a Snapcraft project' -author: 'James Henstridge' -branding: - icon: 'package' - color: 'orange' -inputs: - path: - description: 'The location of the Snapcraft project. Defaults to the base of the repository' - default: '.' - build-info: - description: > - Whether to include build information in the resulting snap. - This will add snap/manifest.yaml and snap/snapcraft.yaml files - to the snap. The Snap Store can use this information to detect - snaps with security vulnerabilities introduced through - stage-packages. - Proprietary applications may want to disable this due to - the information leakage. - default: 'true' - snapcraft-channel: - description: > - The Snapcraft channel to use - By default, the action will use the stable version of Snapcraft - to build the project. This parameter can be used to instead - select a different channel such as beta, candidate, or edge. - default: 'stable' - snapcraft-args: - description: > - Additional arguments to pass to Snapcraft - Some experimental Snapcraft features are disabled by default and - must be turned on via a `--enable-experimental-*` command line - argument. This parameter can be used to turn on such features. - default: '' -outputs: - snap: - description: 'The file name of the resulting snap.' -runs: - using: 'node12' - main: 'dist/index.js' +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: snapcore/action-build@v1 + - uses: snapcore/action-build@v1 + id: snapcraft + - uses: actions/upload-artifact@v2 + with: + name: snap + path: ${{ steps.snapcraft.outputs.snap }} From 8f88b2903f56e5fb6cb44f4d5dce9dcb52053486 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:50:55 -0400 Subject: [PATCH 06/18] Update snapcraftbuild.yml --- .github/workflows/snapcraftbuild.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index cab0660b..50dba94d 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -4,11 +4,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: snapcore/action-build@v1 - - uses: snapcore/action-build@v1 + - uses: actions/checkout + - uses: snapcore/action-build id: snapcraft - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact with: name: snap path: ${{ steps.snapcraft.outputs.snap }} From 3b428f5dd930a62fd468704b5d4be54d860f03bf Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 14:54:03 -0400 Subject: [PATCH 07/18] Update snapcraftbuild.yml --- .github/workflows/snapcraftbuild.yml | 52 +++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index 50dba94d..f36467e8 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -1,13 +1,49 @@ +name: Build and Release Snap + on: - workflow_dispatch: + push: + branches: + - master + jobs: - build: + build-and-release: runs-on: ubuntu-latest + strategy: + matrix: + architecture: + - linux/amd64 + - linux/386 steps: - - uses: actions/checkout - - uses: snapcore/action-build - id: snapcraft - - uses: actions/upload-artifact + - name: Checkout + uses: actions/checkout@v2 + - name: Build + id: build + run: | + echo '{"experimental": true}' | sudo tee /etc/docker/daemon.json > /dev/null + sudo systemctl restart docker + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + ######### Change snap name to temporary test name + sed -i 's/^name:.*/name: opendronemap-core18-test/' snap/snapcraft.yaml + ######### + + docker run --rm --tty --privileged \ + --volume "$GITHUB_WORKSPACE":"$GITHUB_WORKSPACE" \ + --workdir "$GITHUB_WORKSPACE" \ + --platform "${{ matrix.architecture }}" \ + diddledan/snapcraft:core18 + + SNAPFILE="$(find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -name "*.snap" | head -n1)" + sudo chown "$(id -u)":"$(id -g)" "$SNAPFILE" + echo ::set-output name=snap::"$SNAPFILE" + - name: Review + uses: diddlesnaps/snapcraft-review-tools-action@v1 with: - name: snap - path: ${{ steps.snapcraft.outputs.snap }} + snap: ${{ steps.build.outputs.snap }} + isClassic: 'false' + - name: Publish to Edge + uses: snapcore/action-publish@v1 + with: + store_login: ${{ secrets.STORE_LOGIN }} + snap: ${{ steps.build.outputs.snap }} + release: edge From f22b44f99d89fd88b30e7ac240c4888ef9b98e45 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 15:15:24 -0400 Subject: [PATCH 08/18] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8200e8b3..4adedf23 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ ![ODM Logo](https://user-images.githubusercontent.com/1951843/79699889-438ce580-8260-11ea-9c79-8667834aeab2.png) +![Build and Release Snap](https://github.com/Saijin-Naib/ODM/workflows/Build%20and%20Release%20Snap/badge.svg) An open source command line toolkit for processing aerial drone imagery. ODM turns simple 2D images into: From ff3c44d78079fd2a7ec7aece0fb650ac2e1cc86d Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 15:46:47 -0400 Subject: [PATCH 09/18] Update snapcraftbuild.yml Fixes from Dan --- .github/workflows/snapcraftbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index f36467e8..2280b57b 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -31,7 +31,7 @@ jobs: --volume "$GITHUB_WORKSPACE":"$GITHUB_WORKSPACE" \ --workdir "$GITHUB_WORKSPACE" \ --platform "${{ matrix.architecture }}" \ - diddledan/snapcraft:core18 + diddledan/snapcraft:core18 snapcraft --enable-experimental-package-repositories SNAPFILE="$(find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -name "*.snap" | head -n1)" sudo chown "$(id -u)":"$(id -g)" "$SNAPFILE" From 38ae450debebfe835919f53b50e4c50cd6a5cc0d Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 16:00:23 -0400 Subject: [PATCH 10/18] Update snapcraftbuild.yml Dan's rebase on snapcore's action --- .github/workflows/snapcraftbuild.yml | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index 2280b57b..b879e6e6 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -8,40 +8,18 @@ on: jobs: build-and-release: runs-on: ubuntu-latest - strategy: - matrix: - architecture: - - linux/amd64 - - linux/386 steps: - name: Checkout uses: actions/checkout@v2 - name: Build id: build - run: | - echo '{"experimental": true}' | sudo tee /etc/docker/daemon.json > /dev/null - sudo systemctl restart docker - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - ######### Change snap name to temporary test name - sed -i 's/^name:.*/name: opendronemap-core18-test/' snap/snapcraft.yaml - ######### - - docker run --rm --tty --privileged \ - --volume "$GITHUB_WORKSPACE":"$GITHUB_WORKSPACE" \ - --workdir "$GITHUB_WORKSPACE" \ - --platform "${{ matrix.architecture }}" \ - diddledan/snapcraft:core18 snapcraft --enable-experimental-package-repositories - - SNAPFILE="$(find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -name "*.snap" | head -n1)" - sudo chown "$(id -u)":"$(id -g)" "$SNAPFILE" - echo ::set-output name=snap::"$SNAPFILE" + uses: snapcore/action-build@v1 - name: Review uses: diddlesnaps/snapcraft-review-tools-action@v1 with: snap: ${{ steps.build.outputs.snap }} isClassic: 'false' - - name: Publish to Edge + - name: Publish playtest to Edge uses: snapcore/action-publish@v1 with: store_login: ${{ secrets.STORE_LOGIN }} From 1ee4c27d051d22974925011f00256539700159b1 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Thu, 29 Oct 2020 16:08:45 -0400 Subject: [PATCH 11/18] Update snapcraftbuild.yml Fixes from Dan --- .github/workflows/snapcraftbuild.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index b879e6e6..41ad462c 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -14,6 +14,8 @@ jobs: - name: Build id: build uses: snapcore/action-build@v1 + with: + snapcraft-args: --enable-experimental-package-repositories - name: Review uses: diddlesnaps/snapcraft-review-tools-action@v1 with: From e307b43d5ecfd20ad51122634b17fe82f3a3fca0 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Fri, 30 Oct 2020 23:33:19 -0400 Subject: [PATCH 12/18] Update snapcraft.yaml Fixed SNAP package naming --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8e290ba5..6e496d43 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,4 +1,4 @@ -name: opendronemap-core18-test +name: opendronemap adopt-info: odm grade: devel confinement: strict From 1c37ad497a96e3cf16bbdb42a8cf65e6103a9be0 Mon Sep 17 00:00:00 2001 From: Saijin-Naib <19295950+Saijin-Naib@users.noreply.github.com> Date: Fri, 30 Oct 2020 23:34:27 -0400 Subject: [PATCH 13/18] Update README.md Fixed build status badge for where it should live once merged --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4adedf23..9f31aa2d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ ![ODM Logo](https://user-images.githubusercontent.com/1951843/79699889-438ce580-8260-11ea-9c79-8667834aeab2.png) -![Build and Release Snap](https://github.com/Saijin-Naib/ODM/workflows/Build%20and%20Release%20Snap/badge.svg) +![Build and Release Snap](https://github.com/opendronemap/ODM/workflows/Build%20and%20Release%20Snap/badge.svg) An open source command line toolkit for processing aerial drone imagery. ODM turns simple 2D images into: From 9ea1eb6e28034b9b423ad9f017de1786829d870d Mon Sep 17 00:00:00 2001 From: Daniel Llewellyn Date: Mon, 2 Nov 2020 18:35:32 +0000 Subject: [PATCH 14/18] Update build scripts * Use `snapcraft.yaml` as the source-of-truth for APT dependencies to reduce burden of maintianing multiple lists Signed-off-by: Daniel Llewellyn --- Dockerfile | 50 +++++++-- configure.sh | 133 ++++++++++++---------- portable.Dockerfile | 49 +++++++-- snap/snapcraft.yaml | 263 +++++++++++++++++++------------------------- 4 files changed, 265 insertions(+), 230 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca9bb7db..5a7fe96a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:18.04 +FROM ubuntu:18.04 AS builder # Env variables -ENV DEBIAN_FRONTEND noninteractive -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm" -ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" # Prepare directories WORKDIR /code @@ -12,12 +12,9 @@ WORKDIR /code # Copy everything COPY . ./ +# Run the build RUN bash configure.sh install -# Cleanup APT -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # Clean Superbuild RUN rm -rf \ /code/SuperBuild/build/opencv \ @@ -29,5 +26,38 @@ RUN rm -rf \ /code/SuperBuild/src/pcl \ /code/SuperBuild/src/pdal +# find in /code and delete... +RUN find /code \ +# ...*static* libraries... + -type f -name "*.a" -delete \ +# ...and intermediate object files + -or -type f -name "*.o" -delete + +### END Builder + +### Use a second image for the final asset to reduce the number and +# size of the layers. +FROM ubuntu:18.04 + +# Env variables +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" + +WORKDIR /code + +# Copy everything we built from the builder +COPY --from=builder /code /code + +# Copy the Python libraries installed via pip from the builder +COPY --from=builder /usr/local /usr/local + +# Install shared libraries that we depend on via APT, but *not* +# the -dev packages to save space! +RUN bash configure.sh installruntimedepsonly \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # Entry point -ENTRYPOINT ["python3", "/code/run.py"] \ No newline at end of file +ENTRYPOINT ["python3", "/code/run.py"] diff --git a/configure.sh b/configure.sh index 988a4b68..ee222bff 100644 --- a/configure.sh +++ b/configure.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Ensure the DEBIAN_FRONTEND environment variable is set for apt-get calls +APT_GET="env DEBIAN_FRONTEND=noninteractive $(command -v apt-get)" + check_version(){ UBUNTU_VERSION=$(lsb_release -r) if [[ $UBUNTU_VERSION = *"18.04"* ]]; then @@ -21,76 +24,88 @@ else processes=$(nproc) fi +ensure_prereqs() { + export DEBIAN_FRONTEND=noninteractive + + if ! command -v sudo &> /dev/null; then + echo "Installing sudo" + $APT_GET update + $APT_GET install -y -qq --no-install-recommends sudo + else + sudo $APT_GET update + fi + + if ! command -v lsb_release &> /dev/null; then + echo "Installing lsb_release" + sudo $APT_GET install -y -qq --no-install-recommends lsb-release + fi + + if ! command -v pkg-config &> /dev/null; then + echo "Installing pkg-config" + sudo $APT_GET install -y -qq --no-install-recommends pkg-config + fi + + echo "Installing tzdata" + sudo $APT_GET install -y -qq tzdata + + echo "Enabling PPA for Ubuntu GIS" + sudo $APT_GET install -y -qq --no-install-recommends software-properties-common + sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + sudo $APT_GET update + + echo "Installing Python PIP" + sudo $APT_GET install -y -qq --no-install-recommends \ + python3-pip \ + python3-setuptools + sudo pip3 install -U pip + sudo pip3 install -U shyaml +} + +# Save all dependencies in snapcraft.yaml to maintain a single source of truth. +# Maintaining multiple lists will otherwise be painful. +installdepsfromsnapcraft() { + section="$2" + case "$1" in + build) key=build-packages; ;; + runtime) key=stage-packages; ;; + *) key=build-packages; ;; # shouldn't be needed, but it's here just in case + esac + + cat snap/snapcraft.yaml | \ + shyaml get-values-0 parts.$section.$key | \ + xargs -0 sudo $APT_GET install -y -qq --no-install-recommends +} + +installruntimedepsonly() { + echo "Installing runtime dependencies" + ensure_prereqs + check_version + + echo "Installing Required Requisites" + installdepsfromsnapcraft runtime prereqs + echo "Installing OpenCV Dependencies" + installdepsfromsnapcraft runtime opencv + echo "Installing OpenSfM Dependencies" + installdepsfromsnapcraft runtime opensfm +} + install() { cd /code ## Set up library paths - export DEBIAN_FRONTEND=noninteractive export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$RUNPATH/SuperBuild/install/lib ## Before installing echo "Updating the system" - if ! command -v sudo &> /dev/null - then - echo "Installing sudo" - apt-get update && apt-get install -y sudo - fi - sudo apt-get update && sudo apt-get install software-properties-common lsb-release tzdata -y --no-install-recommends - - # Check version + ensure_prereqs check_version - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - sudo apt-get update - echo "Installing Required Requisites" - sudo apt-get install -y -qq --no-install-recommends \ - build-essential \ - git \ - cmake \ - python3-pip \ - libgdal-dev \ - gdal-bin \ - libgeotiff-dev \ - pkg-config \ - libjsoncpp-dev \ - python3-gdal \ - python3-setuptools \ - grass-core \ - libssl-dev \ - swig3.0 \ - python3-wheel \ - libboost-log-dev - sudo pip3 install -U pip - - + installdepsfromsnapcraft build prereqs echo "Installing OpenCV Dependencies" - sudo apt-get install -y -qq --no-install-recommends libgtk2.0-dev \ - libavcodec-dev \ - libavformat-dev \ - libswscale-dev \ - python3-dev \ - libtbb2 \ - libtbb-dev \ - libjpeg-dev \ - libpng-dev \ - libtiff-dev \ - libflann-dev \ - libproj-dev \ - libxext-dev \ - liblapack-dev \ - libeigen3-dev \ - libvtk6-dev - + installdepsfromsnapcraft build opencv echo "Installing OpenSfM Dependencies" - sudo apt-get install -y -qq --no-install-recommends libgoogle-glog-dev \ - libsuitesparse-dev \ - libboost-filesystem-dev \ - libboost-iostreams-dev \ - libboost-regex-dev \ - libboost-python-dev \ - libboost-date-time-dev \ - libboost-thread-dev + installdepsfromsnapcraft build opensfm pip install -r requirements.txt @@ -144,6 +159,8 @@ usage() { echo "Subcommands:" echo " install" echo " Installs all dependencies and modules for running OpenDroneMap" + echo " installruntimedepsonly" + echo " Installs *only* the runtime libraries (used by docker builds). To build from source, use the 'install' command." echo " reinstall" echo " Removes SuperBuild and build modules, then re-installs them. Note this does not update OpenDroneMap to the latest version. " echo " uninstall" @@ -153,7 +170,7 @@ usage() { echo "[nproc] is an optional argument that can set the number of processes for the make -j tag. By default it uses $(nproc)" } -if [[ $1 =~ ^(install|reinstall|uninstall)$ ]]; then +if [[ $1 =~ ^(install|installruntimedepsonly|reinstall|uninstall)$ ]]; then RUNPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" "$1" else diff --git a/portable.Dockerfile b/portable.Dockerfile index b279dc81..b38dbb7b 100644 --- a/portable.Dockerfile +++ b/portable.Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:18.04 +FROM ubuntu:18.04 AS builder # Env variables -ENV DEBIAN_FRONTEND noninteractive -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm" -ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" # Prepare directories WORKDIR /code @@ -12,12 +12,9 @@ WORKDIR /code # Copy everything COPY . ./ +# Run the build RUN PORTABLE_INSTALL=YES bash configure.sh install -# Cleanup APT -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # Clean Superbuild RUN rm -rf \ /code/SuperBuild/build/opencv \ @@ -29,6 +26,38 @@ RUN rm -rf \ /code/SuperBuild/src/pcl \ /code/SuperBuild/src/pdal +# find in /code and delete... +RUN find /code \ +# ...*static* libraries... + -type f -name "*.a" -delete \ +# ...and intermediate object files + -or -type f -name "*.o" -delete + +### END Builder + +### Use a second image for the final asset to reduce the number and +# size of the layers. +FROM ubuntu:18.04 + +# Env variables +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" + +WORKDIR /code + +# Copy everything we built from the builder +COPY --from=builder /code /code + +# Copy the Python libraries installed via pip from the builder +COPY --from=builder /usr/local /usr/local + +# Install shared libraries that we depend on via APT, but *not* +# the -dev packages to save space! +RUN bash configure.sh installruntimedepsonly \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # Entry point ENTRYPOINT ["python3", "/code/run.py"] - diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 6e496d43..5e1e9497 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -30,7 +30,117 @@ package-repositories: ppa: ubuntugis/ubuntugis-unstable parts: + prereqs: + source: . + plugin: nil + override-pull: exit 0 + build-packages: + - build-essential + - cmake + - gdal-bin + - git + - grass-core + - libboost-log-dev + - libgdal-dev + - libgeotiff-dev + - libjsoncpp-dev + - libssl-dev + - libusb-dev + - ninja-build + - pkg-config + - python3-dev + - python3-gdal + - python3-pip + - python3-setuptools + - python3-wheel + - rsync + - swig3.0 + stage-packages: + - gdal-bin + - grass-core + - libboost-log1.65.1 + - libgdal26 + - libgeotiff5 + - libjsoncpp1 + - libspqr2 + - libssl1.1 + - libusb-0.1-4 + - procps + - python3-gdal + - python3-setuptools + stage: + # remove deb-based numpy because it conflicts with our pip-installed version + - -usr/lib/python3/dist-packages/numpy + + opencv: + source: . + plugin: nil + override-pull: exit 0 + build-packages: + - libavcodec-dev + - libavformat-dev + - libeigen3-dev + - libflann-dev + - libgtk2.0-dev + - libjpeg-dev + - liblapack-dev + - libpng-dev + - libproj-dev + - libswscale-dev + - libtbb-dev + - libtiff-dev + - libvtk6-dev + - libxext-dev + stage-packages: + - libavcodec57 + - libavformat57 + - libflann1.9 + - libgtk2.0-0 + - libjpeg-turbo8 + - liblapack3 + - libpng16-16 + - libproj19 + - libswscale4 + - libtbb2 + - libtiff5 + - libvtk6.3 + - libxext6 + + opensfm: + source: . + plugin: nil + override-pull: exit 0 + build-packages: + - libboost-date-time-dev + - libboost-filesystem-dev + - libboost-iostreams-dev + - libboost-python-dev + - libboost-regex-dev + - libboost-thread-dev + - libgoogle-glog-dev + - libsuitesparse-dev + stage-packages: + - libamd2 + - libboost-date-time1.65.1 + - libboost-filesystem1.65.1 + - libboost-iostreams1.65.1 + - libboost-python1.65.1 + - libboost-regex1.65.1 + - libboost-system1.65.1 + - libboost-thread1.65.1 + - libcamd2 + - libccolamd2 + - libcholmod3 + - libcolamd2 + - libcxsparse3 + - libgoogle-glog0v5 + - libsuitesparseconfig5 + odm: + after: + - prereqs + - opencv + - opensfm source: . plugin: nil # We will script everything ourselves build-environment: @@ -62,20 +172,7 @@ parts: $SNAPCRAFT_PART_BUILD/ $SNAPCRAFT_PART_INSTALL/odm/ chmod -R u=rwX,go=rX $PYTHONUSERBASE/lib/python* stage: - # include the libraries and execuables from debian packages - - bin - - lib - - share - - usr/bin - - usr/lib - - usr/share - - # remove deb-based numpy because it conflicts with our pip-installed version - - -usr/lib/python3/dist-packages/numpy - - # include the odm directory ... - - odm - # ... but strip the temporary build files and sources + # strip the temporary build files and sources - -odm/SuperBuild/build/opencv - -odm/SuperBuild/download - -odm/SuperBuild/src/ceres @@ -99,144 +196,6 @@ parts: - -**/*.o build-snaps: - cmake - build-packages: - - build-essential - - gdal-bin - - git - - grass-core - - libboost-log-dev - - libgdal-dev - - libgeotiff-dev - - libjsoncpp-dev - - libssl-dev - - libusb-dev - - ninja-build - - python3-gdal - - python3-pip - - python3-setuptools - - python3-wheel - - rsync - - swig3.0 - # OpenCV - - libavcodec-dev - - libavformat-dev - - libeigen3-dev - - libflann-dev - - libgtk2.0-dev - - libjpeg-dev - - liblapack-dev - - libpng-dev - - libproj-dev - - libswscale-dev - - libtbb-dev - - libtbb2 - - libtiff-dev - - libvtk6-dev - - libxext-dev - - python3-dev - # OpenSfM - - libboost-date-time-dev - - libboost-filesystem-dev - - libboost-iostreams-dev - - libboost-python-dev - - libboost-regex-dev - - libboost-thread-dev - - libgoogle-glog-dev - - libsuitesparse-dev - stage-packages: - - gdal-bin - - grass-core - - libaec0 - - libamd2 - - libasn1-8-heimdal - - libavcodec57 - - libavformat57 - - libboost-filesystem1.65.1 - - libboost-iostreams1.65.1 - - libboost-system1.65.1 - - libboost-thread1.65.1 - - libcamd2 - - libccolamd2 - - libcfitsio5 - - libcharls1 - - libcholmod3 - - libcolamd2 - - libcurl3-gnutls - - libcxsparse3 - - libdap25 - - libdapclient6v5 - - libepsilon1 - - libflann1.9 - - libfontconfig1 - - libfreetype6 - - libfreexl1 - - libfyba0 - - libgdal26 - - libgeos-3.8.0 - - libgeos-c1v5 - - libgeotiff5 - - libgflags2.2 - - libgif7 - - libgomp1 - - libgoogle-glog0v5 - - libgssapi3-heimdal - - libhcrypto4-heimdal - - libhdf4-0-alt - - libhdf5-100 - - libheimbase1-heimdal - - libheimntlm0-heimdal - - libhwloc5 - - libhx509-5-heimdal - - libicu60 - - libjbig0 - - libjpeg-turbo8 - - libjsoncpp1 - - libkmlbase1 - - libkmldom1 - - libkmlengine1 - - libkrb5-26-heimdal - - liblcms2-2 - - libldap-2.4-2 - - libltdl7 - - libmetis5 - - libminizip1 - - libmysqlclient20 - - libnetcdf-c++4 - - libnetcdf13 - - libnghttp2-14 - - libnspr4 - - libnss3 - - libnuma1 - - libodbc1 - - libogdi4.1 - - libopenjp2-7 - - libopenmpi2 - - libpng16-16 - - libpoppler73 - - libpq5 - - libproj19 - - libpsl5 - - libqhull7 - - libroken18-heimdal - - librtmp1 - - libsasl2-2 - - libspatialite7 - - libspqr2 - - libsuitesparseconfig5 - - libsz2 - - libtbb2 - - libtiff5 - - libunwind8 - - liburiparser1 - - libvtk6.3 - - libwebp6 - - libwind0-heimdal - - libxerces-c3.2 - - libxml2 - - odbcinst1debian2 - - procps - - python3-gdal - - python3-setuptools snap-specifics: source: snap/local From 91a82b0f82a528a12834ee005d12e1daafa6e0a6 Mon Sep 17 00:00:00 2001 From: Daniel Llewellyn Date: Mon, 2 Nov 2020 18:36:25 +0000 Subject: [PATCH 15/18] Add test build workflow * Add a GitHub Actions workflow to test that PRs can successfully build Signed-off-by: Daniel Llewellyn --- .github/workflows/test-build-prs.yaml | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/test-build-prs.yaml diff --git a/.github/workflows/test-build-prs.yaml b/.github/workflows/test-build-prs.yaml new file mode 100644 index 00000000..3db3982d --- /dev/null +++ b/.github/workflows/test-build-prs.yaml @@ -0,0 +1,43 @@ +name: Build PRs + +on: + pull-request: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build + uses: docker/build-push-action@v2 + with: + file: ./portable.Dockerfile + platforms: linux/amd64,linux/386 + push: false + + snapcraft: + runs-on: ubuntu-latest + strategy: + matrix: + architecture: + - i386 + - amd64 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build + id: build + uses: diddlesnaps/snapcraft-multiarch-action@v1 + with: + snapcraft-args: --enable-experimental-package-repositories + architecture: ${{ matrix.architecture }} + - name: Review + uses: diddlesnaps/snapcraft-review-tools-action@v1 + with: + snap: ${{ steps.build.outputs.snap }} + isClassic: 'false' From 41819dccb93f07bf3e5cf81be89835ad1064f6e3 Mon Sep 17 00:00:00 2001 From: Daniel Llewellyn Date: Mon, 2 Nov 2020 18:38:06 +0000 Subject: [PATCH 16/18] Add build and release workflow * Add build and release workflow to build a Docker container and WSL rootfs tarball. * Upload the Docker container to Docker Hub. * Upload the WSL rootfs tarball to GitHub Releases. Signed-off-by: Daniel Llewellyn --- .../workflows/build-container-and-wsl.yaml | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/build-container-and-wsl.yaml diff --git a/.github/workflows/build-container-and-wsl.yaml b/.github/workflows/build-container-and-wsl.yaml new file mode 100644 index 00000000..91412847 --- /dev/null +++ b/.github/workflows/build-container-and-wsl.yaml @@ -0,0 +1,70 @@ +name: Build and Release Container and WSL Image + +on: + push: + tags: + - v* + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + # Use the repository information of the checked-out code to format docker tags + - name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + images: opendronemap/odm + tag_match: \d{1,3}.\d{1,3}.\d{1,3} # matches v1.2.3 + - name: Build and push Docker image + id: docker_build + uses: docker/build-push-action@v2 + with: + file: ./portable.Dockerfile + platforms: linux/amd64,linux/386 + push: true + load: true # Copy to the host's docker daemon so we can use it for WSL export + tags: ${{ steps.docker_meta.outputs.tags }} + - name: Export WSL image + id: wsl_export + run: | + docker export $(docker create opendronemap/odm) --output odm-wsl-rootfs-amd64.tar.gz + gzip odm-wsl-rootfs-amd64.tar.gz + echo ::set-output name=amd64-rootfs::"odm-wsl-rootfs-amd64.tar.gz" + # Convert tag into a GitHub Release + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false + # Upload the WSL image to the new Release + - name: Upload amd64 Release Asset + id: upload-amd64-wsl-rootfs + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./${{ steps.wsl_export.outputs.amd64-rootfs }} + asset_name: ${{ steps.wsl_export.outputs.amd64-rootfs }} + asset_content_type: application/gzip + - name: Docker image digest and WSL rootfs download URL + run: | + echo "Docker image digest: ${{ steps.docker_build.outputs.digest }}" + echo "WSL AMD64 rootfs URL: ${{ steps.upload-amd64-wsl-rootfs.browser_download_url }}" From d49f22c89882ad70079e29ca20b9e49720a19f39 Mon Sep 17 00:00:00 2001 From: Daniel Llewellyn Date: Mon, 2 Nov 2020 19:05:14 +0000 Subject: [PATCH 17/18] Update snapcraftbuild workflow * Use newly published action: `diddlesnaps/snapcraft-multiarch-action` * Upload every push to master to the Snap Store's Edge channel * Upload every tagged prerelease to the Snap Store's Beta channel (including release candidates which are also handled below) * Upload every tagged stable or release candidate release to the Snap Store's Candidate Channel Signed-off-by: Daniel Llewellyn --- .github/workflows/snapcraftbuild.yml | 29 ++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index 41ad462c..c0686530 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -4,26 +4,51 @@ on: push: branches: - master + tags: + - v** jobs: build-and-release: runs-on: ubuntu-latest + strategy: + matrix: + architecture: + - i386 + - amd64 steps: - name: Checkout uses: actions/checkout@v2 - name: Build id: build - uses: snapcore/action-build@v1 + uses: diddlesnaps/snapcraft-multiarch-action@v1 with: snapcraft-args: --enable-experimental-package-repositories + architecture: ${{ matrix.architecture }} - name: Review uses: diddlesnaps/snapcraft-review-tools-action@v1 with: snap: ${{ steps.build.outputs.snap }} isClassic: 'false' - - name: Publish playtest to Edge + - name: Publish unstable builds to Edge + if: ${{ github.ref }} == "refs/heads/master" uses: snapcore/action-publish@v1 with: store_login: ${{ secrets.STORE_LOGIN }} snap: ${{ steps.build.outputs.snap }} release: edge + - name: Publish tagged prerelease builds to Beta + # These are identified by having a hyphen in the tag name, e.g.: v1.0.0-beta1 + if: ${{ startsWith("refs/tags/v", github.ref) }} && ${{ contains("-", github.ref) }} + uses: snapcore/action-publish@v1 + with: + store_login: ${{ secrets.STORE_LOGIN }} + snap: ${{ steps.build.outputs.snap }} + release: beta + - name: Publish tagged stable or release-candidate builds to Candidate + # These are identified by NOT having a hyphen in the tag name, OR having "-RC" or "-rc" in the tag name. + if: ${{ startsWith("refs/tags/v", github.ref) }} && ( ( ! ${{ contains("-", github.ref) }} ) || ${{ contains("-RC", github.ref) }} || ${{ contains("-rc", github.ref) }} ) + uses: snapcore/action-publish@v1 + with: + store_login: ${{ secrets.STORE_LOGIN }} + snap: ${{ steps.build.outputs.snap }} + release: candidate From 149baa436d5f773a1b4481f127072d43488405a4 Mon Sep 17 00:00:00 2001 From: Daniel Llewellyn Date: Mon, 2 Nov 2020 19:11:19 +0000 Subject: [PATCH 18/18] Fix if: lines in snapcraftbuild workflow * Incorrect syntax for expressions used in `if:` tests in `snapcraftbuild.yml` workflow Signed-off-by: Daniel Llewellyn --- .github/workflows/snapcraftbuild.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/snapcraftbuild.yml b/.github/workflows/snapcraftbuild.yml index c0686530..b61e02c2 100644 --- a/.github/workflows/snapcraftbuild.yml +++ b/.github/workflows/snapcraftbuild.yml @@ -30,7 +30,7 @@ jobs: snap: ${{ steps.build.outputs.snap }} isClassic: 'false' - name: Publish unstable builds to Edge - if: ${{ github.ref }} == "refs/heads/master" + if: github.ref == "refs/heads/master" uses: snapcore/action-publish@v1 with: store_login: ${{ secrets.STORE_LOGIN }} @@ -38,7 +38,7 @@ jobs: release: edge - name: Publish tagged prerelease builds to Beta # These are identified by having a hyphen in the tag name, e.g.: v1.0.0-beta1 - if: ${{ startsWith("refs/tags/v", github.ref) }} && ${{ contains("-", github.ref) }} + if: startsWith("refs/tags/v", github.ref) && contains("-", github.ref) uses: snapcore/action-publish@v1 with: store_login: ${{ secrets.STORE_LOGIN }} @@ -46,7 +46,7 @@ jobs: release: beta - name: Publish tagged stable or release-candidate builds to Candidate # These are identified by NOT having a hyphen in the tag name, OR having "-RC" or "-rc" in the tag name. - if: ${{ startsWith("refs/tags/v", github.ref) }} && ( ( ! ${{ contains("-", github.ref) }} ) || ${{ contains("-RC", github.ref) }} || ${{ contains("-rc", github.ref) }} ) + if: startsWith("refs/tags/v", github.ref) && ( ( ! contains("-", github.ref) ) || contains("-RC", github.ref) || contains("-rc", github.ref) ) uses: snapcore/action-publish@v1 with: store_login: ${{ secrets.STORE_LOGIN }}