Update build scripts

* Use `snapcraft.yaml` as the source-of-truth for APT dependencies to reduce burden of maintianing multiple lists Signed-off-by: Daniel Llewellyn <daniel@snapcraft.ninja>
2020-11-02 18:35:32 +00:00 · 2020-11-02 18:35:32 +00:00 · 9ea1eb6e28
commit 9ea1eb6e28
--- a/50
+++ b/50
@ -1,10 +1,10 @@
-FROM ubuntu:18.04
+FROM ubuntu:18.04 AS builder

 # Env variables
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages"
-ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm"
-ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"

 # Prepare directories
 WORKDIR /code
@ -12,12 +12,9 @@ WORKDIR /code
 # Copy everything
 COPY . ./

+# Run the build
 RUN bash configure.sh install

-# Cleanup APT
-RUN apt-get clean \
-  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
-
 # Clean Superbuild
 RUN rm -rf \
  /code/SuperBuild/build/opencv \
@ -29,5 +26,38 @@ RUN rm -rf \
  /code/SuperBuild/src/pcl \
  /code/SuperBuild/src/pdal

+# find in /code and delete...
+RUN find /code \
+# ...*static* libraries...
+  -type f -name "*.a" -delete \
+# ...and intermediate object files
+  -or -type f -name "*.o" -delete
+
+### END Builder
+
+### Use a second image for the final asset to reduce the number and
+# size of the layers.
+FROM ubuntu:18.04
+
+# Env variables
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
+
+WORKDIR /code
+
+# Copy everything we built from the builder
+COPY --from=builder /code /code
+
+# Copy the Python libraries installed via pip from the builder
+COPY --from=builder /usr/local /usr/local
+
+# Install shared libraries that we depend on via APT, but *not*
+# the -dev packages to save space!
+RUN bash configure.sh installruntimedepsonly \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
+
 # Entry point
-ENTRYPOINT ["python3", "/code/run.py"]
+ENTRYPOINT ["python3", "/code/run.py"]
--- a/configure.sh
+++ b/configure.sh
@ -1,5 +1,8 @@
 #!/bin/bash

+# Ensure the DEBIAN_FRONTEND environment variable is set for apt-get calls
+APT_GET="env DEBIAN_FRONTEND=noninteractive $(command -v apt-get)"
+
 check_version(){
  UBUNTU_VERSION=$(lsb_release -r)
  if [[ $UBUNTU_VERSION = *"18.04"* ]]; then
@ -21,76 +24,88 @@ else
    processes=$(nproc)
 fi

+ensure_prereqs() {
+    export DEBIAN_FRONTEND=noninteractive
+
+    if ! command -v sudo &> /dev/null; then
+        echo "Installing sudo"
+        $APT_GET update
+        $APT_GET install -y -qq --no-install-recommends sudo
+    else
+        sudo $APT_GET update
+    fi
+
+    if ! command -v lsb_release &> /dev/null; then
+        echo "Installing lsb_release"
+        sudo $APT_GET install -y -qq --no-install-recommends lsb-release
+    fi
+
+    if ! command -v pkg-config &> /dev/null; then
+        echo "Installing pkg-config"
+        sudo $APT_GET install -y -qq --no-install-recommends pkg-config
+    fi
+
+    echo "Installing tzdata"
+    sudo $APT_GET install -y -qq tzdata
+
+    echo "Enabling PPA for Ubuntu GIS"
+    sudo $APT_GET install -y -qq --no-install-recommends software-properties-common
+    sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
+    sudo $APT_GET update
+
+    echo "Installing Python PIP"
+    sudo $APT_GET install -y -qq --no-install-recommends \
+        python3-pip \
+        python3-setuptools
+    sudo pip3 install -U pip
+    sudo pip3 install -U shyaml
+}
+
+# Save all dependencies in snapcraft.yaml to maintain a single source of truth.
+# Maintaining multiple lists will otherwise be painful.
+installdepsfromsnapcraft() {
+    section="$2"
+    case "$1" in
+        build) key=build-packages; ;;
+        runtime) key=stage-packages; ;;
+        *) key=build-packages; ;; # shouldn't be needed, but it's here just in case
+    esac
+
+    cat snap/snapcraft.yaml | \
+        shyaml get-values-0 parts.$section.$key | \
+        xargs -0 sudo $APT_GET install -y -qq --no-install-recommends
+}
+
+installruntimedepsonly() {
+    echo "Installing runtime dependencies"
+    ensure_prereqs
+    check_version
+
+    echo "Installing Required Requisites"
+    installdepsfromsnapcraft runtime prereqs
+    echo "Installing OpenCV Dependencies"
+    installdepsfromsnapcraft runtime opencv
+    echo "Installing OpenSfM Dependencies"
+    installdepsfromsnapcraft runtime opensfm
+}
+    
 install() {
    cd /code
    
    ## Set up library paths
-    export DEBIAN_FRONTEND=noninteractive
    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$RUNPATH/SuperBuild/install/lib

 	## Before installing
    echo "Updating the system"
-    if ! command -v sudo &> /dev/null
-    then
-        echo "Installing sudo"
-        apt-get update && apt-get install -y sudo
-    fi
-    sudo apt-get update && sudo apt-get install software-properties-common lsb-release tzdata -y  --no-install-recommends
-    
-    # Check version
+    ensure_prereqs
    check_version
    
-    sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
-    sudo apt-get update
-    
    echo "Installing Required Requisites"
-    sudo apt-get install -y -qq --no-install-recommends \
-                         build-essential \
-                         git \
-                         cmake \
-                         python3-pip \
-                         libgdal-dev \
-                         gdal-bin \
-                         libgeotiff-dev \
-                         pkg-config \
-                         libjsoncpp-dev \
-                         python3-gdal \
-                         python3-setuptools \
-                         grass-core \
-                         libssl-dev \
-                         swig3.0 \
-                         python3-wheel \
-                         libboost-log-dev
-    sudo pip3 install -U pip
-
-
+    installdepsfromsnapcraft build prereqs
    echo "Installing OpenCV Dependencies"
-    sudo apt-get install -y -qq --no-install-recommends libgtk2.0-dev \
-                         libavcodec-dev \
-                         libavformat-dev \
-                         libswscale-dev \
-                         python3-dev \
-                         libtbb2 \
-                         libtbb-dev \
-                         libjpeg-dev \
-                         libpng-dev \
-                         libtiff-dev \
-                         libflann-dev \
-                         libproj-dev \
-                         libxext-dev \
-                         liblapack-dev \
-                         libeigen3-dev \
-                         libvtk6-dev
-	
+    installdepsfromsnapcraft build opencv
    echo "Installing OpenSfM Dependencies"
-    sudo apt-get install -y -qq  --no-install-recommends libgoogle-glog-dev \
-                         libsuitesparse-dev \
-                         libboost-filesystem-dev \
-                         libboost-iostreams-dev \
-                         libboost-regex-dev \
-                         libboost-python-dev \
-                         libboost-date-time-dev \
-                         libboost-thread-dev
+    installdepsfromsnapcraft build opensfm

    pip install -r requirements.txt

@ -144,6 +159,8 @@ usage() {
    echo "Subcommands:"
    echo "  install"
    echo "    Installs all dependencies and modules for running OpenDroneMap"
+    echo "  installruntimedepsonly"
+    echo "    Installs *only* the runtime libraries (used by docker builds). To build from source, use the 'install' command."
    echo "  reinstall"
    echo "    Removes SuperBuild and build modules, then re-installs them. Note this does not update OpenDroneMap to the latest version. "
    echo "  uninstall"
@ -153,7 +170,7 @@ usage() {
    echo "[nproc] is an optional argument that can set the number of processes for the make -j tag. By default it uses $(nproc)"
 }

-if [[ $1 =~ ^(install|reinstall|uninstall)$ ]]; then
+if [[ $1 =~ ^(install|installruntimedepsonly|reinstall|uninstall)$ ]]; then
    RUNPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
    "$1"
 else
--- a/portable.Dockerfile
+++ b/portable.Dockerfile
@ -1,10 +1,10 @@
-FROM ubuntu:18.04
+FROM ubuntu:18.04 AS builder

 # Env variables
-ENV DEBIAN_FRONTEND noninteractive
-ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages"
-ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm"
-ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"

 # Prepare directories
 WORKDIR /code
@ -12,12 +12,9 @@ WORKDIR /code
 # Copy everything
 COPY . ./

+# Run the build
 RUN PORTABLE_INSTALL=YES bash configure.sh install

-# Cleanup APT
-RUN apt-get clean \
-  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
-
 # Clean Superbuild
 RUN rm -rf \
  /code/SuperBuild/build/opencv \
@ -29,6 +26,38 @@ RUN rm -rf \
  /code/SuperBuild/src/pcl \
  /code/SuperBuild/src/pdal

+# find in /code and delete...
+RUN find /code \
+# ...*static* libraries...
+  -type f -name "*.a" -delete \
+# ...and intermediate object files
+  -or -type f -name "*.o" -delete
+
+### END Builder
+
+### Use a second image for the final asset to reduce the number and
+# size of the layers.
+FROM ubuntu:18.04
+
+# Env variables
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
+
+WORKDIR /code
+
+# Copy everything we built from the builder
+COPY --from=builder /code /code
+
+# Copy the Python libraries installed via pip from the builder
+COPY --from=builder /usr/local /usr/local
+
+# Install shared libraries that we depend on via APT, but *not*
+# the -dev packages to save space!
+RUN bash configure.sh installruntimedepsonly \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 # Entry point
 ENTRYPOINT ["python3", "/code/run.py"]
-
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@ -30,7 +30,117 @@ package-repositories:
    ppa: ubuntugis/ubuntugis-unstable

 parts:
+  prereqs:
+    source: .
+    plugin: nil
+    override-pull: exit 0
+    build-packages:
+      - build-essential
+      - cmake
+      - gdal-bin
+      - git
+      - grass-core
+      - libboost-log-dev
+      - libgdal-dev
+      - libgeotiff-dev
+      - libjsoncpp-dev
+      - libssl-dev
+      - libusb-dev
+      - ninja-build
+      - pkg-config
+      - python3-dev
+      - python3-gdal
+      - python3-pip
+      - python3-setuptools
+      - python3-wheel
+      - rsync
+      - swig3.0
+    stage-packages:
+      - gdal-bin
+      - grass-core
+      - libboost-log1.65.1
+      - libgdal26
+      - libgeotiff5
+      - libjsoncpp1
+      - libspqr2
+      - libssl1.1
+      - libusb-0.1-4
+      - procps
+      - python3-gdal
+      - python3-setuptools
+    stage:
+      # remove deb-based numpy because it conflicts with our pip-installed version
+      - -usr/lib/python3/dist-packages/numpy
+
+  opencv:
+    source: .
+    plugin: nil
+    override-pull: exit 0
+    build-packages:
+      - libavcodec-dev
+      - libavformat-dev
+      - libeigen3-dev
+      - libflann-dev
+      - libgtk2.0-dev
+      - libjpeg-dev
+      - liblapack-dev
+      - libpng-dev
+      - libproj-dev
+      - libswscale-dev
+      - libtbb-dev
+      - libtiff-dev
+      - libvtk6-dev
+      - libxext-dev
+    stage-packages:
+      - libavcodec57
+      - libavformat57
+      - libflann1.9
+      - libgtk2.0-0
+      - libjpeg-turbo8
+      - liblapack3
+      - libpng16-16
+      - libproj19
+      - libswscale4
+      - libtbb2
+      - libtiff5
+      - libvtk6.3
+      - libxext6
+
+  opensfm:
+    source: .
+    plugin: nil
+    override-pull: exit 0
+    build-packages:
+      - libboost-date-time-dev
+      - libboost-filesystem-dev
+      - libboost-iostreams-dev
+      - libboost-python-dev
+      - libboost-regex-dev
+      - libboost-thread-dev
+      - libgoogle-glog-dev
+      - libsuitesparse-dev
+    stage-packages:
+      - libamd2
+      - libboost-date-time1.65.1
+      - libboost-filesystem1.65.1
+      - libboost-iostreams1.65.1
+      - libboost-python1.65.1
+      - libboost-regex1.65.1
+      - libboost-system1.65.1
+      - libboost-thread1.65.1
+      - libcamd2
+      - libccolamd2
+      - libcholmod3
+      - libcolamd2
+      - libcxsparse3
+      - libgoogle-glog0v5
+      - libsuitesparseconfig5
+
  odm:
+    after:
+      - prereqs
+      - opencv
+      - opensfm
    source: .
    plugin: nil # We will script everything ourselves
    build-environment:
@ -62,20 +172,7 @@ parts:
        $SNAPCRAFT_PART_BUILD/ $SNAPCRAFT_PART_INSTALL/odm/
      chmod -R u=rwX,go=rX $PYTHONUSERBASE/lib/python*
    stage:
-      # include the libraries and execuables from debian packages
-      - bin
-      - lib
-      - share
-      - usr/bin
-      - usr/lib
-      - usr/share
-
-      # remove deb-based numpy because it conflicts with our pip-installed version
-      - -usr/lib/python3/dist-packages/numpy
-
-      # include the odm directory ...
-      - odm
-      # ... but strip the temporary build files and sources
+      # strip the temporary build files and sources
      - -odm/SuperBuild/build/opencv
      - -odm/SuperBuild/download
      - -odm/SuperBuild/src/ceres
@ -99,144 +196,6 @@ parts:
      - -**/*.o
    build-snaps:
      - cmake
-    build-packages:
-      - build-essential
-      - gdal-bin
-      - git
-      - grass-core
-      - libboost-log-dev
-      - libgdal-dev
-      - libgeotiff-dev
-      - libjsoncpp-dev
-      - libssl-dev
-      - libusb-dev
-      - ninja-build
-      - python3-gdal
-      - python3-pip
-      - python3-setuptools
-      - python3-wheel
-      - rsync
-      - swig3.0
-      # OpenCV
-      - libavcodec-dev
-      - libavformat-dev
-      - libeigen3-dev
-      - libflann-dev
-      - libgtk2.0-dev
-      - libjpeg-dev
-      - liblapack-dev
-      - libpng-dev
-      - libproj-dev
-      - libswscale-dev
-      - libtbb-dev
-      - libtbb2
-      - libtiff-dev
-      - libvtk6-dev
-      - libxext-dev
-      - python3-dev
-      # OpenSfM
-      - libboost-date-time-dev
-      - libboost-filesystem-dev
-      - libboost-iostreams-dev
-      - libboost-python-dev
-      - libboost-regex-dev
-      - libboost-thread-dev
-      - libgoogle-glog-dev
-      - libsuitesparse-dev
-    stage-packages:
-      - gdal-bin
-      - grass-core
-      - libaec0
-      - libamd2
-      - libasn1-8-heimdal
-      - libavcodec57
-      - libavformat57
-      - libboost-filesystem1.65.1
-      - libboost-iostreams1.65.1
-      - libboost-system1.65.1
-      - libboost-thread1.65.1
-      - libcamd2
-      - libccolamd2
-      - libcfitsio5
-      - libcharls1
-      - libcholmod3
-      - libcolamd2
-      - libcurl3-gnutls
-      - libcxsparse3
-      - libdap25
-      - libdapclient6v5
-      - libepsilon1
-      - libflann1.9
-      - libfontconfig1
-      - libfreetype6
-      - libfreexl1
-      - libfyba0
-      - libgdal26
-      - libgeos-3.8.0
-      - libgeos-c1v5
-      - libgeotiff5
-      - libgflags2.2
-      - libgif7
-      - libgomp1
-      - libgoogle-glog0v5
-      - libgssapi3-heimdal
-      - libhcrypto4-heimdal
-      - libhdf4-0-alt
-      - libhdf5-100
-      - libheimbase1-heimdal
-      - libheimntlm0-heimdal
-      - libhwloc5
-      - libhx509-5-heimdal
-      - libicu60
-      - libjbig0
-      - libjpeg-turbo8
-      - libjsoncpp1
-      - libkmlbase1
-      - libkmldom1
-      - libkmlengine1
-      - libkrb5-26-heimdal
-      - liblcms2-2
-      - libldap-2.4-2
-      - libltdl7
-      - libmetis5
-      - libminizip1
-      - libmysqlclient20
-      - libnetcdf-c++4
-      - libnetcdf13
-      - libnghttp2-14
-      - libnspr4
-      - libnss3
-      - libnuma1
-      - libodbc1
-      - libogdi4.1
-      - libopenjp2-7
-      - libopenmpi2
-      - libpng16-16
-      - libpoppler73
-      - libpq5
-      - libproj19
-      - libpsl5
-      - libqhull7
-      - libroken18-heimdal
-      - librtmp1
-      - libsasl2-2
-      - libspatialite7
-      - libspqr2
-      - libsuitesparseconfig5
-      - libsz2
-      - libtbb2
-      - libtiff5
-      - libunwind8
-      - liburiparser1
-      - libvtk6.3
-      - libwebp6
-      - libwind0-heimdal
-      - libxerces-c3.2
-      - libxml2
-      - odbcinst1debian2
-      - procps
-      - python3-gdal
-      - python3-setuptools

  snap-specifics:
    source: snap/local