diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 834540e2..c7d43034 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -32,6 +32,7 @@ jobs: uses: docker/build-push-action@v2 with: file: ./Dockerfile + platforms: linux/amd64,linux/arm64 push: true tags: | ${{ steps.docker_meta.outputs.tags }} diff --git a/Dockerfile b/Dockerfile index fcd21b35..f9ec3dfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM ubuntu:21.04 MAINTAINER Piero Toffanin ENV PYTHONUNBUFFERED 1 @@ -6,46 +6,29 @@ ENV PYTHONPATH $PYTHONPATH:/webodm ENV PROJ_LIB=/usr/share/proj # Prepare directory -RUN mkdir /webodm +ADD . /webodm/ WORKDIR /webodm -RUN apt-get -qq update && apt-get install -y software-properties-common tzdata -RUN add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable - # Install Node.js -RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl -RUN wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh -RUN apt-get -qq update && apt-get -qq install -y nodejs - -# Install Python3, GDAL, nginx, letsencrypt, psql -RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-12 gettext -RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2 - -# Install pip reqs -ADD requirements.txt /webodm/ -RUN pip install -r requirements.txt - -ADD . /webodm/ - -# Setup cron -RUN ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh - -#RUN git submodule update --init - -WORKDIR /webodm/nodeodm/external/NodeODM -RUN npm install --quiet - -WORKDIR /webodm -RUN npm install --quiet -g webpack@4.16.5 && npm install --quiet -g webpack-cli@4.2.0 && npm install --quiet && webpack --mode production -RUN echo "UTC" > /etc/timezone -RUN python manage.py collectstatic --noinput -RUN python manage.py rebuildplugins -RUN python manage.py translate build --safe - -# Cleanup -RUN apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -RUN rm /webodm/webodm/secret_key.py +RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \ + wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh && \ + apt-get -qq update && apt-get -qq install -y nodejs && \ + # Install Python3, GDAL, nginx, letsencrypt, psql + apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin libgdal-dev python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-13 gettext tzdata && \ + update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \ + # Install pip reqs + pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \ + # Setup cron + ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh && \ + cd /webodm/nodeodm/external/NodeODM && npm install --quiet && cd /webodm && \ + npm install --quiet -g webpack@4.16.5 && npm install --quiet -g webpack-cli@4.2.0 && npm install --quiet && webpack --mode production && \ + echo "UTC" > /etc/timezone && \ + python manage.py collectstatic --noinput && \ + python manage.py rebuildplugins && \ + python manage.py translate build --safe && \ + # Cleanup + apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + rm /webodm/webodm/secret_key.py VOLUME /webodm/app/media diff --git a/db/Dockerfile b/db/Dockerfile index 47d7278f..e026e1d2 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -1,18 +1,83 @@ -FROM postgres:9.5 +FROM ubuntu:20.04 MAINTAINER Piero Toffanin -ENV POSTGIS_MAJOR 2.3 ENV POSTGRES_PASSWORD postgres +ENV POSTGRES_HOST_AUTH_METHOD trust +ENV GOSU_VERSION 1.12 +ENV PG_MAJOR 9.5 +ENV PG_VERSION 9.5.25 +ENV POSTGIS_VERSION 2.3.2 +ENV PATH $PATH:/usr/local/pgsql/bin +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG en_US.utf8 +ENV PGDATA /var/lib/postgresql/data -RUN . /etc/os-release \ - && echo "deb http://deb.debian.org/debian "$VERSION_CODENAME"-backports main" >> /etc/apt/sources.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - postgis \ - && rm -rf /var/lib/apt/lists/* - -EXPOSE 5432 +RUN mkdir /docker-entrypoint-initdb.d COPY init.sql /docker-entrypoint-initdb.d/init-db.sql RUN chmod 644 /docker-entrypoint-initdb.d/init-db.sql + +COPY docker-entrypoint.sh /usr/local/bin/ +RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat + +# Setup system +RUN apt-get update; \ + set -eux; \ + groupadd -r postgres --gid=999; \ + useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \ + mkdir -p /var/lib/postgresql; \ + chown -R postgres:postgres /var/lib/postgresql; \ + # grab gosu for easy step-down from root + # https://github.com/tianon/gosu/releases + apt-get update; \ + apt-get install -y --no-install-recommends wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget --no-check-certificate -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true; \ + # make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default + apt-get update; \ + apt-get install -y --no-install-recommends locales; \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8; \ + # Build Postgres from source + mkdir /staging; \ + apt-get update; \ + apt-get install -y --no-install-recommends wget gcc build-essential libproj-dev libgeos-dev libxml2-dev zlib1g-dev libreadline-dev; \ + cd /staging; \ + wget --no-check-certificate -q https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.2/postgresql-$PG_VERSION.tar.gz; \ + cd /staging; \ + tar -zxf postgresql-$PG_VERSION.tar.gz; \ + cd postgresql-$PG_VERSION; \ + ./configure; \ + make -j$(nproc); \ + make install; \ + postgres --version; \ + sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/pgsql/share/postgresql.conf.sample; \ + grep -F "listen_addresses = '*'" /usr/local/pgsql/share/postgresql.conf.sample; \ + mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql; \ + # this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values) + mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"; \ + # Build PostGIS from source + apt-get update; \ + apt-get install -y --no-install-recommends libgdal-dev libjson-c-dev; \ + cd /staging; \ + wget --no-check-certificate -q https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.2/postgis-$POSTGIS_VERSION.tar.gz; \ + wget --no-check-certificate -q -O /usr/include/json-c/json_object_private.h https://raw.githubusercontent.com/json-c/json-c/json-c-0.13/json_object_private.h; \ + tar -zxf postgis-$POSTGIS_VERSION.tar.gz; \ + sed -i 's/#error.*/#define ACCEPT_USE_OF_DEPRECATED_PROJ_API_H 1/' /usr/include/proj_api.h; \ + cd /staging/postgis-$POSTGIS_VERSION; \ + ./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config; \ + make; \ + make install; \ + sed -i '1d' /usr/local/pgsql/share/extension/postgis--$POSTGIS_VERSION.sql; \ + apt-get remove -y gcc build-essential wget; \ + apt-get autoremove -y; \ + apt-get clean; \ + rm -fr /var/lib/apt/lists/* /staging /usr/include; + +VOLUME /var/lib/postgresql/data +ENTRYPOINT ["docker-entrypoint.sh"] +STOPSIGNAL SIGINT +EXPOSE 5432 +CMD ["postgres"] \ No newline at end of file diff --git a/db/docker-entrypoint.sh b/db/docker-entrypoint.sh new file mode 100755 index 00000000..9778e898 --- /dev/null +++ b/db/docker-entrypoint.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "$POSTGRES_INITDB_XLOGDIR" ]; then + mkdir -p "$POSTGRES_INITDB_XLOGDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_XLOGDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_XLOGDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then + export LD_PRELOAD='/usr/lib/libnss_wrapper.so' + export NSS_WRAPPER_PASSWD="$(mktemp)" + export NSS_WRAPPER_GROUP="$(mktemp)" + echo "postgres:x:$(id -u):$(id -g):PostgreSQL:$PGDATA:/bin/false" > "$NSS_WRAPPER_PASSWD" + echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP" + fi + + if [ -n "$POSTGRES_INITDB_XLOGDIR" ]; then + set -- --xlogdir "$POSTGRES_INITDB_XLOGDIR" "$@" + fi + + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; docker_process_sql -f "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) echo "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo + else + echo + echo 'PostgreSQL Database directory appears to contain a database; Skipping initialization' + echo + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi \ No newline at end of file diff --git a/db/init.sql b/db/init.sql index 03ea75d3..dc83e41b 100644 --- a/db/init.sql +++ b/db/init.sql @@ -2,4 +2,3 @@ ALTER USER postgres PASSWORD 'postgres'; CREATE DATABASE webodm_dev; ALTER DATABASE webodm_dev SET postgis.gdal_enabled_drivers TO 'GTiff'; ALTER DATABASE webodm_dev SET postgis.enable_outdb_rasters TO True; - diff --git a/requirements.txt b/requirements.txt index 63012fc6..ca4605f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ gunicorn==19.7.1 itypes==1.1.0 kombu==4.6.7 libsass==0.19.4 -Markdown==2.6.7 +Markdown==3.3.4 olefile==0.44 openapi-codec==1.1.7 packaging==16.8 @@ -62,3 +62,4 @@ https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp37-cp37m-win_amd64 Shapely==1.7.0 ; sys_platform == "win32" eventlet==0.25.1 ; sys_platform == "win32" pyopenssl==19.1.0 ; sys_platform == "win32" +numpy==1.21.1 \ No newline at end of file