Merge pull request #1048 from pierotofy/m1

M1 Changes
pull/1054/head
Piero Toffanin 2021-09-03 10:24:19 -04:00 zatwierdzone przez GitHub
commit 2f474501b2
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
6 zmienionych plików z 429 dodań i 53 usunięć

Wyświetl plik

@ -32,6 +32,7 @@ jobs:
uses: docker/build-push-action@v2
with:
file: ./Dockerfile
platforms: linux/amd64,linux/arm64
push: true
tags: |
${{ steps.docker_meta.outputs.tags }}

Wyświetl plik

@ -1,4 +1,4 @@
FROM ubuntu:20.04
FROM ubuntu:21.04
MAINTAINER Piero Toffanin <pt@masseranolabs.com>
ENV PYTHONUNBUFFERED 1
@ -6,46 +6,29 @@ ENV PYTHONPATH $PYTHONPATH:/webodm
ENV PROJ_LIB=/usr/share/proj
# Prepare directory
RUN mkdir /webodm
ADD . /webodm/
WORKDIR /webodm
RUN apt-get -qq update && apt-get install -y software-properties-common tzdata
RUN add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
# Install Node.js
RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl
RUN wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh
RUN apt-get -qq update && apt-get -qq install -y nodejs
RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \
wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh && \
apt-get -qq update && apt-get -qq install -y nodejs && \
# Install Python3, GDAL, nginx, letsencrypt, psql
RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-12 gettext
RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin libgdal-dev python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-13 gettext tzdata && \
update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \
# Install pip reqs
ADD requirements.txt /webodm/
RUN pip install -r requirements.txt
ADD . /webodm/
pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \
# Setup cron
RUN ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh
#RUN git submodule update --init
WORKDIR /webodm/nodeodm/external/NodeODM
RUN npm install --quiet
WORKDIR /webodm
RUN npm install --quiet -g webpack@4.16.5 && npm install --quiet -g webpack-cli@4.2.0 && npm install --quiet && webpack --mode production
RUN echo "UTC" > /etc/timezone
RUN python manage.py collectstatic --noinput
RUN python manage.py rebuildplugins
RUN python manage.py translate build --safe
ln -s /webodm/nginx/crontab /var/spool/cron/crontabs/root && chmod 0644 /webodm/nginx/crontab && service cron start && chmod +x /webodm/nginx/letsencrypt-autogen.sh && \
cd /webodm/nodeodm/external/NodeODM && npm install --quiet && cd /webodm && \
npm install --quiet -g webpack@4.16.5 && npm install --quiet -g webpack-cli@4.2.0 && npm install --quiet && webpack --mode production && \
echo "UTC" > /etc/timezone && \
python manage.py collectstatic --noinput && \
python manage.py rebuildplugins && \
python manage.py translate build --safe && \
# Cleanup
RUN apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN rm /webodm/webodm/secret_key.py
apt-get remove -y g++ python3-dev libpq-dev && apt-get autoremove -y && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
rm /webodm/webodm/secret_key.py
VOLUME /webodm/app/media

Wyświetl plik

@ -1,18 +1,83 @@
FROM postgres:9.5
FROM ubuntu:20.04
MAINTAINER Piero Toffanin <pt@masseranolabs.com>
ENV POSTGIS_MAJOR 2.3
ENV POSTGRES_PASSWORD postgres
ENV POSTGRES_HOST_AUTH_METHOD trust
ENV GOSU_VERSION 1.12
ENV PG_MAJOR 9.5
ENV PG_VERSION 9.5.25
ENV POSTGIS_VERSION 2.3.2
ENV PATH $PATH:/usr/local/pgsql/bin
ENV DEBIAN_FRONTEND=noninteractive
ENV LANG en_US.utf8
ENV PGDATA /var/lib/postgresql/data
RUN . /etc/os-release \
&& echo "deb http://deb.debian.org/debian "$VERSION_CODENAME"-backports main" >> /etc/apt/sources.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \
postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \
postgis \
&& rm -rf /var/lib/apt/lists/*
EXPOSE 5432
RUN mkdir /docker-entrypoint-initdb.d
COPY init.sql /docker-entrypoint-initdb.d/init-db.sql
RUN chmod 644 /docker-entrypoint-initdb.d/init-db.sql
COPY docker-entrypoint.sh /usr/local/bin/
RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
# Setup system
RUN apt-get update; \
set -eux; \
groupadd -r postgres --gid=999; \
useradd -r -g postgres --uid=999 --home-dir=/var/lib/postgresql --shell=/bin/bash postgres; \
mkdir -p /var/lib/postgresql; \
chown -R postgres:postgres /var/lib/postgresql; \
# grab gosu for easy step-down from root
# https://github.com/tianon/gosu/releases
apt-get update; \
apt-get install -y --no-install-recommends wget; \
rm -rf /var/lib/apt/lists/*; \
dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
wget --no-check-certificate -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
chmod +x /usr/local/bin/gosu; \
gosu --version; \
gosu nobody true; \
# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
apt-get update; \
apt-get install -y --no-install-recommends locales; \
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8; \
# Build Postgres from source
mkdir /staging; \
apt-get update; \
apt-get install -y --no-install-recommends wget gcc build-essential libproj-dev libgeos-dev libxml2-dev zlib1g-dev libreadline-dev; \
cd /staging; \
wget --no-check-certificate -q https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.2/postgresql-$PG_VERSION.tar.gz; \
cd /staging; \
tar -zxf postgresql-$PG_VERSION.tar.gz; \
cd postgresql-$PG_VERSION; \
./configure; \
make -j$(nproc); \
make install; \
postgres --version; \
sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/pgsql/share/postgresql.conf.sample; \
grep -F "listen_addresses = '*'" /usr/local/pgsql/share/postgresql.conf.sample; \
mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2777 /var/run/postgresql; \
# this 777 will be replaced by 700 at runtime (allows semi-arbitrary "--user" values)
mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"; \
# Build PostGIS from source
apt-get update; \
apt-get install -y --no-install-recommends libgdal-dev libjson-c-dev; \
cd /staging; \
wget --no-check-certificate -q https://github.com/OpenDroneMap/WebODM/releases/download/v1.9.2/postgis-$POSTGIS_VERSION.tar.gz; \
wget --no-check-certificate -q -O /usr/include/json-c/json_object_private.h https://raw.githubusercontent.com/json-c/json-c/json-c-0.13/json_object_private.h; \
tar -zxf postgis-$POSTGIS_VERSION.tar.gz; \
sed -i 's/#error.*/#define ACCEPT_USE_OF_DEPRECATED_PROJ_API_H 1/' /usr/include/proj_api.h; \
cd /staging/postgis-$POSTGIS_VERSION; \
./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config; \
make; \
make install; \
sed -i '1d' /usr/local/pgsql/share/extension/postgis--$POSTGIS_VERSION.sql; \
apt-get remove -y gcc build-essential wget; \
apt-get autoremove -y; \
apt-get clean; \
rm -fr /var/lib/apt/lists/* /staging /usr/include;
VOLUME /var/lib/postgresql/data
ENTRYPOINT ["docker-entrypoint.sh"]
STOPSIGNAL SIGINT
EXPOSE 5432
CMD ["postgres"]

Wyświetl plik

@ -0,0 +1,327 @@
#!/usr/bin/env bash
set -Eeo pipefail
# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables)
# usage: file_env VAR [DEFAULT]
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
export "$var"="$val"
unset "$fileVar"
}
# check to see if this file is being run or sourced from another script
_is_sourced() {
# https://unix.stackexchange.com/a/215279
[ "${#FUNCNAME[@]}" -ge 2 ] \
&& [ "${FUNCNAME[0]}" = '_is_sourced' ] \
&& [ "${FUNCNAME[1]}" = 'source' ]
}
# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user
docker_create_db_directories() {
local user; user="$(id -u)"
mkdir -p "$PGDATA"
# ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory)
chmod 700 "$PGDATA" || :
# ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289
mkdir -p /var/run/postgresql || :
chmod 775 /var/run/postgresql || :
# Create the transaction log directory before initdb is run so the directory is owned by the correct user
if [ -n "$POSTGRES_INITDB_XLOGDIR" ]; then
mkdir -p "$POSTGRES_INITDB_XLOGDIR"
if [ "$user" = '0' ]; then
find "$POSTGRES_INITDB_XLOGDIR" \! -user postgres -exec chown postgres '{}' +
fi
chmod 700 "$POSTGRES_INITDB_XLOGDIR"
fi
# allow the container to be started with `--user`
if [ "$user" = '0' ]; then
find "$PGDATA" \! -user postgres -exec chown postgres '{}' +
find /var/run/postgresql \! -user postgres -exec chown postgres '{}' +
fi
}
# initialize empty PGDATA directory with new database via 'initdb'
# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function
# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames
# this is also where the database user is created, specified by `POSTGRES_USER` env
docker_init_database_dir() {
# "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary
# see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html
if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then
export LD_PRELOAD='/usr/lib/libnss_wrapper.so'
export NSS_WRAPPER_PASSWD="$(mktemp)"
export NSS_WRAPPER_GROUP="$(mktemp)"
echo "postgres:x:$(id -u):$(id -g):PostgreSQL:$PGDATA:/bin/false" > "$NSS_WRAPPER_PASSWD"
echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP"
fi
if [ -n "$POSTGRES_INITDB_XLOGDIR" ]; then
set -- --xlogdir "$POSTGRES_INITDB_XLOGDIR" "$@"
fi
eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"'
# unset/cleanup "nss_wrapper" bits
if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then
rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP"
unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP
fi
}
# print large warning if POSTGRES_PASSWORD is long
# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust'
# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust'
# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ]
docker_verify_minimum_env() {
# check password first so we can output the warning before postgres
# messes it up
if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then
cat >&2 <<-'EOWARN'
WARNING: The supplied POSTGRES_PASSWORD is 100+ characters.
This will not work if used via PGPASSWORD with "psql".
https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412)
https://github.com/docker-library/postgres/issues/507
EOWARN
fi
if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then
# The - option suppresses leading tabs but *not* spaces. :)
cat >&2 <<-'EOE'
Error: Database is uninitialized and superuser password is not specified.
You must specify POSTGRES_PASSWORD to a non-empty value for the
superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run".
You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all
connections without a password. This is *not* recommended.
See PostgreSQL documentation about "trust":
https://www.postgresql.org/docs/current/auth-trust.html
EOE
exit 1
fi
if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then
cat >&2 <<-'EOWARN'
********************************************************************************
WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow
anyone with access to the Postgres port to access your database without
a password, even if POSTGRES_PASSWORD is set. See PostgreSQL
documentation about "trust":
https://www.postgresql.org/docs/current/auth-trust.html
In Docker's default configuration, this is effectively any other
container on the same system.
It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace
it with "-e POSTGRES_PASSWORD=password" instead to set a password in
"docker run".
********************************************************************************
EOWARN
fi
}
# usage: docker_process_init_files [file [file [...]]]
# ie: docker_process_init_files /always-initdb.d/*
# process initializer files, based on file extensions and permissions
docker_process_init_files() {
# psql here for backwards compatibility "${psql[@]}"
psql=( docker_process_sql )
echo
local f
for f; do
case "$f" in
*.sh)
# https://github.com/docker-library/postgres/issues/450#issuecomment-393167936
# https://github.com/docker-library/postgres/pull/452
if [ -x "$f" ]; then
echo "$0: running $f"
"$f"
else
echo "$0: sourcing $f"
. "$f"
fi
;;
*.sql) echo "$0: running $f"; docker_process_sql -f "$f"; echo ;;
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;;
*.sql.xz) echo "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;;
*) echo "$0: ignoring $f" ;;
esac
echo
done
}
# Execute sql script, passed via stdin (or -f flag of pqsl)
# usage: docker_process_sql [psql-cli-args]
# ie: docker_process_sql --dbname=mydb <<<'INSERT ...'
# ie: docker_process_sql -f my-file.sql
# ie: docker_process_sql <my-file.sql
docker_process_sql() {
local query_runner=( psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password )
if [ -n "$POSTGRES_DB" ]; then
query_runner+=( --dbname "$POSTGRES_DB" )
fi
PGHOST= PGHOSTADDR= "${query_runner[@]}" "$@"
}
# create initial database
# uses environment variables for input: POSTGRES_DB
docker_setup_db() {
local dbAlreadyExists
dbAlreadyExists="$(
POSTGRES_DB= docker_process_sql --dbname postgres --set db="$POSTGRES_DB" --tuples-only <<-'EOSQL'
SELECT 1 FROM pg_database WHERE datname = :'db' ;
EOSQL
)"
if [ -z "$dbAlreadyExists" ]; then
POSTGRES_DB= docker_process_sql --dbname postgres --set db="$POSTGRES_DB" <<-'EOSQL'
CREATE DATABASE :"db" ;
EOSQL
echo
fi
}
# Loads various settings that are used elsewhere in the script
# This should be called before any other functions
docker_setup_env() {
file_env 'POSTGRES_PASSWORD'
file_env 'POSTGRES_USER' 'postgres'
file_env 'POSTGRES_DB' "$POSTGRES_USER"
file_env 'POSTGRES_INITDB_ARGS'
# default authentication method is md5
: "${POSTGRES_HOST_AUTH_METHOD:=md5}"
declare -g DATABASE_ALREADY_EXISTS
# look specifically for PG_VERSION, as it is expected in the DB dir
if [ -s "$PGDATA/PG_VERSION" ]; then
DATABASE_ALREADY_EXISTS='true'
fi
}
# append POSTGRES_HOST_AUTH_METHOD to pg_hba.conf for "host" connections
pg_setup_hba_conf() {
{
echo
if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then
echo '# warning trust is enabled for all connections'
echo '# see https://www.postgresql.org/docs/12/auth-trust.html'
fi
echo "host all all all $POSTGRES_HOST_AUTH_METHOD"
} >> "$PGDATA/pg_hba.conf"
}
# start socket-only postgresql server for setting up or running scripts
# all arguments will be passed along as arguments to `postgres` (via pg_ctl)
docker_temp_server_start() {
if [ "$1" = 'postgres' ]; then
shift
fi
# internal start of server in order to allow setup using psql client
# does not listen on external TCP/IP and waits until start finishes
set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}"
PGUSER="${PGUSER:-$POSTGRES_USER}" \
pg_ctl -D "$PGDATA" \
-o "$(printf '%q ' "$@")" \
-w start
}
# stop postgresql server after done setting up user and running scripts
docker_temp_server_stop() {
PGUSER="${PGUSER:-postgres}" \
pg_ctl -D "$PGDATA" -m fast -w stop
}
# check arguments for an option that would cause postgres to stop
# return true if there is one
_pg_want_help() {
local arg
for arg; do
case "$arg" in
# postgres --help | grep 'then exit'
# leaving out -C on purpose since it always fails and is unhelpful:
# postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory
-'?'|--help|--describe-config|-V|--version)
return 0
;;
esac
done
return 1
}
_main() {
# if first arg looks like a flag, assume we want to run postgres server
if [ "${1:0:1}" = '-' ]; then
set -- postgres "$@"
fi
if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then
docker_setup_env
# setup data directories and permissions (when run as root)
docker_create_db_directories
if [ "$(id -u)" = '0' ]; then
# then restart script as postgres user
exec gosu postgres "$BASH_SOURCE" "$@"
fi
# only run initialization on an empty data directory
if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
docker_verify_minimum_env
# check dir permissions to reduce likelihood of half-initialized database
ls /docker-entrypoint-initdb.d/ > /dev/null
docker_init_database_dir
pg_setup_hba_conf
# PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless
# e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS
export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}"
docker_temp_server_start "$@"
docker_setup_db
docker_process_init_files /docker-entrypoint-initdb.d/*
docker_temp_server_stop
unset PGPASSWORD
echo
echo 'PostgreSQL init process complete; ready for start up.'
echo
else
echo
echo 'PostgreSQL Database directory appears to contain a database; Skipping initialization'
echo
fi
fi
exec "$@"
}
if ! _is_sourced; then
_main "$@"
fi

Wyświetl plik

@ -2,4 +2,3 @@ ALTER USER postgres PASSWORD 'postgres';
CREATE DATABASE webodm_dev;
ALTER DATABASE webodm_dev SET postgis.gdal_enabled_drivers TO 'GTiff';
ALTER DATABASE webodm_dev SET postgis.enable_outdb_rasters TO True;

Wyświetl plik

@ -26,7 +26,7 @@ gunicorn==19.7.1
itypes==1.1.0
kombu==4.6.7
libsass==0.19.4
Markdown==2.6.7
Markdown==3.3.4
olefile==0.44
openapi-codec==1.1.7
packaging==16.8
@ -62,3 +62,4 @@ https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp37-cp37m-win_amd64
Shapely==1.7.0 ; sys_platform == "win32"
eventlet==0.25.1 ; sys_platform == "win32"
pyopenssl==19.1.0 ; sys_platform == "win32"
numpy==1.21.1