diff --git a/Dockerfile b/Dockerfile index ca9bb7db..5a7fe96a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:18.04 +FROM ubuntu:18.04 AS builder # Env variables -ENV DEBIAN_FRONTEND noninteractive -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm" -ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" # Prepare directories WORKDIR /code @@ -12,12 +12,9 @@ WORKDIR /code # Copy everything COPY . ./ +# Run the build RUN bash configure.sh install -# Cleanup APT -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # Clean Superbuild RUN rm -rf \ /code/SuperBuild/build/opencv \ @@ -29,5 +26,38 @@ RUN rm -rf \ /code/SuperBuild/src/pcl \ /code/SuperBuild/src/pdal +# find in /code and delete... +RUN find /code \ +# ...*static* libraries... + -type f -name "*.a" -delete \ +# ...and intermediate object files + -or -type f -name "*.o" -delete + +### END Builder + +### Use a second image for the final asset to reduce the number and +# size of the layers. +FROM ubuntu:18.04 + +# Env variables +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" + +WORKDIR /code + +# Copy everything we built from the builder +COPY --from=builder /code /code + +# Copy the Python libraries installed via pip from the builder +COPY --from=builder /usr/local /usr/local + +# Install shared libraries that we depend on via APT, but *not* +# the -dev packages to save space! +RUN bash configure.sh installruntimedepsonly \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # Entry point -ENTRYPOINT ["python3", "/code/run.py"] \ No newline at end of file +ENTRYPOINT ["python3", "/code/run.py"] diff --git a/configure.sh b/configure.sh index 988a4b68..ee222bff 100644 --- a/configure.sh +++ b/configure.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Ensure the DEBIAN_FRONTEND environment variable is set for apt-get calls +APT_GET="env DEBIAN_FRONTEND=noninteractive $(command -v apt-get)" + check_version(){ UBUNTU_VERSION=$(lsb_release -r) if [[ $UBUNTU_VERSION = *"18.04"* ]]; then @@ -21,76 +24,88 @@ else processes=$(nproc) fi +ensure_prereqs() { + export DEBIAN_FRONTEND=noninteractive + + if ! command -v sudo &> /dev/null; then + echo "Installing sudo" + $APT_GET update + $APT_GET install -y -qq --no-install-recommends sudo + else + sudo $APT_GET update + fi + + if ! command -v lsb_release &> /dev/null; then + echo "Installing lsb_release" + sudo $APT_GET install -y -qq --no-install-recommends lsb-release + fi + + if ! command -v pkg-config &> /dev/null; then + echo "Installing pkg-config" + sudo $APT_GET install -y -qq --no-install-recommends pkg-config + fi + + echo "Installing tzdata" + sudo $APT_GET install -y -qq tzdata + + echo "Enabling PPA for Ubuntu GIS" + sudo $APT_GET install -y -qq --no-install-recommends software-properties-common + sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + sudo $APT_GET update + + echo "Installing Python PIP" + sudo $APT_GET install -y -qq --no-install-recommends \ + python3-pip \ + python3-setuptools + sudo pip3 install -U pip + sudo pip3 install -U shyaml +} + +# Save all dependencies in snapcraft.yaml to maintain a single source of truth. +# Maintaining multiple lists will otherwise be painful. +installdepsfromsnapcraft() { + section="$2" + case "$1" in + build) key=build-packages; ;; + runtime) key=stage-packages; ;; + *) key=build-packages; ;; # shouldn't be needed, but it's here just in case + esac + + cat snap/snapcraft.yaml | \ + shyaml get-values-0 parts.$section.$key | \ + xargs -0 sudo $APT_GET install -y -qq --no-install-recommends +} + +installruntimedepsonly() { + echo "Installing runtime dependencies" + ensure_prereqs + check_version + + echo "Installing Required Requisites" + installdepsfromsnapcraft runtime prereqs + echo "Installing OpenCV Dependencies" + installdepsfromsnapcraft runtime opencv + echo "Installing OpenSfM Dependencies" + installdepsfromsnapcraft runtime opensfm +} + install() { cd /code ## Set up library paths - export DEBIAN_FRONTEND=noninteractive export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$RUNPATH/SuperBuild/install/lib ## Before installing echo "Updating the system" - if ! command -v sudo &> /dev/null - then - echo "Installing sudo" - apt-get update && apt-get install -y sudo - fi - sudo apt-get update && sudo apt-get install software-properties-common lsb-release tzdata -y --no-install-recommends - - # Check version + ensure_prereqs check_version - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - sudo apt-get update - echo "Installing Required Requisites" - sudo apt-get install -y -qq --no-install-recommends \ - build-essential \ - git \ - cmake \ - python3-pip \ - libgdal-dev \ - gdal-bin \ - libgeotiff-dev \ - pkg-config \ - libjsoncpp-dev \ - python3-gdal \ - python3-setuptools \ - grass-core \ - libssl-dev \ - swig3.0 \ - python3-wheel \ - libboost-log-dev - sudo pip3 install -U pip - - + installdepsfromsnapcraft build prereqs echo "Installing OpenCV Dependencies" - sudo apt-get install -y -qq --no-install-recommends libgtk2.0-dev \ - libavcodec-dev \ - libavformat-dev \ - libswscale-dev \ - python3-dev \ - libtbb2 \ - libtbb-dev \ - libjpeg-dev \ - libpng-dev \ - libtiff-dev \ - libflann-dev \ - libproj-dev \ - libxext-dev \ - liblapack-dev \ - libeigen3-dev \ - libvtk6-dev - + installdepsfromsnapcraft build opencv echo "Installing OpenSfM Dependencies" - sudo apt-get install -y -qq --no-install-recommends libgoogle-glog-dev \ - libsuitesparse-dev \ - libboost-filesystem-dev \ - libboost-iostreams-dev \ - libboost-regex-dev \ - libboost-python-dev \ - libboost-date-time-dev \ - libboost-thread-dev + installdepsfromsnapcraft build opensfm pip install -r requirements.txt @@ -144,6 +159,8 @@ usage() { echo "Subcommands:" echo " install" echo " Installs all dependencies and modules for running OpenDroneMap" + echo " installruntimedepsonly" + echo " Installs *only* the runtime libraries (used by docker builds). To build from source, use the 'install' command." echo " reinstall" echo " Removes SuperBuild and build modules, then re-installs them. Note this does not update OpenDroneMap to the latest version. " echo " uninstall" @@ -153,7 +170,7 @@ usage() { echo "[nproc] is an optional argument that can set the number of processes for the make -j tag. By default it uses $(nproc)" } -if [[ $1 =~ ^(install|reinstall|uninstall)$ ]]; then +if [[ $1 =~ ^(install|installruntimedepsonly|reinstall|uninstall)$ ]]; then RUNPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" "$1" else diff --git a/portable.Dockerfile b/portable.Dockerfile index b279dc81..b38dbb7b 100644 --- a/portable.Dockerfile +++ b/portable.Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:18.04 +FROM ubuntu:18.04 AS builder # Env variables -ENV DEBIAN_FRONTEND noninteractive -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" -ENV PYTHONPATH "$PYTHONPATH:/code/SuperBuild/src/opensfm" -ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" # Prepare directories WORKDIR /code @@ -12,12 +12,9 @@ WORKDIR /code # Copy everything COPY . ./ +# Run the build RUN PORTABLE_INSTALL=YES bash configure.sh install -# Cleanup APT -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # Clean Superbuild RUN rm -rf \ /code/SuperBuild/build/opencv \ @@ -29,6 +26,38 @@ RUN rm -rf \ /code/SuperBuild/src/pcl \ /code/SuperBuild/src/pdal +# find in /code and delete... +RUN find /code \ +# ...*static* libraries... + -type f -name "*.a" -delete \ +# ...and intermediate object files + -or -type f -name "*.o" -delete + +### END Builder + +### Use a second image for the final asset to reduce the number and +# size of the layers. +FROM ubuntu:18.04 + +# Env variables +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.6/dist-packages" \ + PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" \ + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" + +WORKDIR /code + +# Copy everything we built from the builder +COPY --from=builder /code /code + +# Copy the Python libraries installed via pip from the builder +COPY --from=builder /usr/local /usr/local + +# Install shared libraries that we depend on via APT, but *not* +# the -dev packages to save space! +RUN bash configure.sh installruntimedepsonly \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # Entry point ENTRYPOINT ["python3", "/code/run.py"] - diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 6e496d43..5e1e9497 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -30,7 +30,117 @@ package-repositories: ppa: ubuntugis/ubuntugis-unstable parts: + prereqs: + source: . + plugin: nil + override-pull: exit 0 + build-packages: + - build-essential + - cmake + - gdal-bin + - git + - grass-core + - libboost-log-dev + - libgdal-dev + - libgeotiff-dev + - libjsoncpp-dev + - libssl-dev + - libusb-dev + - ninja-build + - pkg-config + - python3-dev + - python3-gdal + - python3-pip + - python3-setuptools + - python3-wheel + - rsync + - swig3.0 + stage-packages: + - gdal-bin + - grass-core + - libboost-log1.65.1 + - libgdal26 + - libgeotiff5 + - libjsoncpp1 + - libspqr2 + - libssl1.1 + - libusb-0.1-4 + - procps + - python3-gdal + - python3-setuptools + stage: + # remove deb-based numpy because it conflicts with our pip-installed version + - -usr/lib/python3/dist-packages/numpy + + opencv: + source: . + plugin: nil + override-pull: exit 0 + build-packages: + - libavcodec-dev + - libavformat-dev + - libeigen3-dev + - libflann-dev + - libgtk2.0-dev + - libjpeg-dev + - liblapack-dev + - libpng-dev + - libproj-dev + - libswscale-dev + - libtbb-dev + - libtiff-dev + - libvtk6-dev + - libxext-dev + stage-packages: + - libavcodec57 + - libavformat57 + - libflann1.9 + - libgtk2.0-0 + - libjpeg-turbo8 + - liblapack3 + - libpng16-16 + - libproj19 + - libswscale4 + - libtbb2 + - libtiff5 + - libvtk6.3 + - libxext6 + + opensfm: + source: . + plugin: nil + override-pull: exit 0 + build-packages: + - libboost-date-time-dev + - libboost-filesystem-dev + - libboost-iostreams-dev + - libboost-python-dev + - libboost-regex-dev + - libboost-thread-dev + - libgoogle-glog-dev + - libsuitesparse-dev + stage-packages: + - libamd2 + - libboost-date-time1.65.1 + - libboost-filesystem1.65.1 + - libboost-iostreams1.65.1 + - libboost-python1.65.1 + - libboost-regex1.65.1 + - libboost-system1.65.1 + - libboost-thread1.65.1 + - libcamd2 + - libccolamd2 + - libcholmod3 + - libcolamd2 + - libcxsparse3 + - libgoogle-glog0v5 + - libsuitesparseconfig5 + odm: + after: + - prereqs + - opencv + - opensfm source: . plugin: nil # We will script everything ourselves build-environment: @@ -62,20 +172,7 @@ parts: $SNAPCRAFT_PART_BUILD/ $SNAPCRAFT_PART_INSTALL/odm/ chmod -R u=rwX,go=rX $PYTHONUSERBASE/lib/python* stage: - # include the libraries and execuables from debian packages - - bin - - lib - - share - - usr/bin - - usr/lib - - usr/share - - # remove deb-based numpy because it conflicts with our pip-installed version - - -usr/lib/python3/dist-packages/numpy - - # include the odm directory ... - - odm - # ... but strip the temporary build files and sources + # strip the temporary build files and sources - -odm/SuperBuild/build/opencv - -odm/SuperBuild/download - -odm/SuperBuild/src/ceres @@ -99,144 +196,6 @@ parts: - -**/*.o build-snaps: - cmake - build-packages: - - build-essential - - gdal-bin - - git - - grass-core - - libboost-log-dev - - libgdal-dev - - libgeotiff-dev - - libjsoncpp-dev - - libssl-dev - - libusb-dev - - ninja-build - - python3-gdal - - python3-pip - - python3-setuptools - - python3-wheel - - rsync - - swig3.0 - # OpenCV - - libavcodec-dev - - libavformat-dev - - libeigen3-dev - - libflann-dev - - libgtk2.0-dev - - libjpeg-dev - - liblapack-dev - - libpng-dev - - libproj-dev - - libswscale-dev - - libtbb-dev - - libtbb2 - - libtiff-dev - - libvtk6-dev - - libxext-dev - - python3-dev - # OpenSfM - - libboost-date-time-dev - - libboost-filesystem-dev - - libboost-iostreams-dev - - libboost-python-dev - - libboost-regex-dev - - libboost-thread-dev - - libgoogle-glog-dev - - libsuitesparse-dev - stage-packages: - - gdal-bin - - grass-core - - libaec0 - - libamd2 - - libasn1-8-heimdal - - libavcodec57 - - libavformat57 - - libboost-filesystem1.65.1 - - libboost-iostreams1.65.1 - - libboost-system1.65.1 - - libboost-thread1.65.1 - - libcamd2 - - libccolamd2 - - libcfitsio5 - - libcharls1 - - libcholmod3 - - libcolamd2 - - libcurl3-gnutls - - libcxsparse3 - - libdap25 - - libdapclient6v5 - - libepsilon1 - - libflann1.9 - - libfontconfig1 - - libfreetype6 - - libfreexl1 - - libfyba0 - - libgdal26 - - libgeos-3.8.0 - - libgeos-c1v5 - - libgeotiff5 - - libgflags2.2 - - libgif7 - - libgomp1 - - libgoogle-glog0v5 - - libgssapi3-heimdal - - libhcrypto4-heimdal - - libhdf4-0-alt - - libhdf5-100 - - libheimbase1-heimdal - - libheimntlm0-heimdal - - libhwloc5 - - libhx509-5-heimdal - - libicu60 - - libjbig0 - - libjpeg-turbo8 - - libjsoncpp1 - - libkmlbase1 - - libkmldom1 - - libkmlengine1 - - libkrb5-26-heimdal - - liblcms2-2 - - libldap-2.4-2 - - libltdl7 - - libmetis5 - - libminizip1 - - libmysqlclient20 - - libnetcdf-c++4 - - libnetcdf13 - - libnghttp2-14 - - libnspr4 - - libnss3 - - libnuma1 - - libodbc1 - - libogdi4.1 - - libopenjp2-7 - - libopenmpi2 - - libpng16-16 - - libpoppler73 - - libpq5 - - libproj19 - - libpsl5 - - libqhull7 - - libroken18-heimdal - - librtmp1 - - libsasl2-2 - - libspatialite7 - - libspqr2 - - libsuitesparseconfig5 - - libsz2 - - libtbb2 - - libtiff5 - - libunwind8 - - liburiparser1 - - libvtk6.3 - - libwebp6 - - libwind0-heimdal - - libxerces-c3.2 - - libxml2 - - odbcinst1debian2 - - procps - - python3-gdal - - python3-setuptools snap-specifics: source: snap/local