Merge pull request #175 from pierotofy/gunicorn

Gunicorn
pull/198/head
Piero Toffanin 2017-05-24 09:46:50 -04:00 zatwierdzone przez GitHub
commit ba4078f8c2
13 zmienionych plików z 193 dodań i 26 usunięć

Wyświetl plik

@ -26,8 +26,8 @@ RUN printf "Package: *\nPin: release a=testing\nPin-Priority: 750\n" > /etc/apt/
RUN printf "deb http://mirror.steadfast.net/debian/ stable main contrib non-free\ndeb-src http://mirror.steadfast.net/debian/ stable main contrib non-free" > /etc/apt/sources.list.d/stable.list
RUN printf "deb http://mirror.steadfast.net/debian/ testing main contrib non-free\ndeb-src http://mirror.steadfast.net/debian/ testing main contrib non-free" > /etc/apt/sources.list.d/testing.list
# Install GDAL
RUN apt-get update && apt-get install -t testing -y binutils libproj-dev gdal-bin
# Install GDAL, nginx
RUN apt-get update && apt-get install -t testing -y binutils libproj-dev gdal-bin nginx
WORKDIR /webodm/nodeodm/external/node-OpenDroneMap
RUN npm install

Wyświetl plik

@ -136,17 +136,22 @@ screen -r webodm
## Run it natively
If you want to run WebODM natively, you will need to install:
WebODM can run natively on Windows, MacOS and Linux.
Ubuntu 16.04 LTS users can refer to [this script](/contrib/ubuntu_1604_install.sh) to install WebODM natively on a new machine.
To run WebODM, you will need to install:
* PostgreSQL (>= 9.5)
* PostGIS 2.3
* Python 3.5
* GDAL (>= 2.1)
* Node.js (>= 6.0)
* Nginx (Linux/MacOS) - OR - Apache + mod_wsgi (Windows)
On Linux, make sure you have:
```bash
apt-get install binutils libproj-dev gdal-bin
apt-get install binutils libproj-dev gdal-bin nginx
```
On Windows use the [OSGeo4W](https://trac.osgeo.org/osgeo4w/) installer to install GDAL. MacOS users can use:
@ -190,7 +195,17 @@ pip install -r requirements.txt
sudo npm install -g webpack
npm install
webpack
chmod +x start.sh && ./start.sh
chmod +x start.sh && ./start.sh --no-gunicorn
```
The `start.sh` script will use Django's built-in server if you pass the `--no-gunicorn` parameter. This is good for testing, but bad for production.
In production, if you have nginx installed, modify the configuration file in `nginx/nginx.conf` to match your system's configuration and just run `start.sh` without parameters.
Windows users should refer to [this guide](https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/modwsgi/) to install Apache + mod_wsgi and run gunicorn:
```bash
gunicorn webodm.wsgi --bind 0.0.0.0:8000 --preload
```
If you are getting a `rt_raster_gdal_warp: Could not create GDAL transformation object for output dataset creation`, make sure that your PostGIS installation has PROJ support:
@ -218,8 +233,6 @@ gdalinfo --version
```
Should all work without errors.
Ubuntu 16.04 LTS users can refer to this [script](https://gist.githubusercontent.com/lkpanganiban/5226cc8dd59cb39cdc1946259c3fea6e/raw/f9f41ad0c1dfdd2d26a452d3b2732dbaf3fd3608/webodm_install.sh) to install WebODM natively on a new machine.
## OpenDroneMap, node-OpenDroneMap, WebODM... what?
The [OpenDroneMap project](https://github.com/OpenDroneMap/) is composed of several components.

Wyświetl plik

@ -20,10 +20,10 @@ class ProcessingNodeSerializer(serializers.ModelSerializer):
fields = '__all__'
class ProcessingNodeFilter(FilterSet):
has_available_options = django_filters.MethodFilter()
has_available_options = django_filters.CharFilter(method='filter_has_available_options')
# noinspection PyMethodMayBeStatic
def filter_has_available_options(self, queryset, value):
def filter_has_available_options(self, queryset, name, value):
if value.lower() in ['true', '1']:
return queryset.exclude(available_options=dict())
else:

Wyświetl plik

@ -26,5 +26,5 @@ class ProjectViewSet(viewsets.ModelViewSet):
"""
filter_fields = ('id', 'name', 'description', 'created_at')
serializer_class = ProjectSerializer
queryset = models.Project.objects.filter(deleting=False)
queryset = models.Project.objects.filter(deleting=False).order_by('-created_at')
ordering_fields = '__all__'

Wyświetl plik

@ -1,4 +1,3 @@
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User, Group
from django.core.exceptions import ObjectDoesNotExist
@ -6,14 +5,25 @@ from django.db.utils import ProgrammingError
from guardian.shortcuts import assign_perm
from nodeodm.models import ProcessingNode
from . import signals, scheduler
import logging, os
# noinspection PyUnresolvedReferences
from . import scheduler, signals
import logging
from .models import Task
from webodm import settings
from webodm.wsgi import booted
def boot():
# booted is a shared memory variable to keep track of boot status
# as multiple workers could trigger the boot sequence twice
if not settings.DEBUG and booted.value: return
booted.value = True
logger = logging.getLogger('app.logger')
logger.info("Booting...")
if settings.DEBUG:
logger.warning("Debug mode is ON (for development this is OK)")
# Check default group
try:

1
build/.gitignore vendored 100644
Wyświetl plik

@ -0,0 +1 @@
*

Wyświetl plik

@ -0,0 +1,62 @@
# Configure system libs
sudo apt-get autoremove -y install-info
sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable
sudo apt-get update
sudo apt-get install -y python-dev libpq-dev gdal-bin libgdal-dev libproj-dev python-virtualenv python3-dev
# Setup Postgres
ppaexists=$( grep ^ /etc/apt/sources.list /etc/apt/sources.list.d/* | grep postgres )
if [ ! $ppaexists ]; then
echo "Add PostgreSQL PPA..."
add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main"
sudo wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
apt-get update
else
echo "PostgreSQL PPA already exists..."
fi
sudo apt-get install -y postgresql-9.6
sudo apt-get install -y postgresql-9.6-postgis-2.3
sudo apt-get install -y python-psycopg2
sudo -u postgres bash -c "psql -c \"CREATE USER postgres WITH PASSWORD 'postgres';\""
sudo -u postgres bash -c "psql -c \"ALTER ROLE postgres WITH SUPERUSER;\""
sudo -u postgres createdb -O postgres webodm_dev -E utf-8
sudo -u postgres bash -c "psql -d webodm_dev -c \"CREATE EXTENSION postgis;\""
sudo -u postgres bash -c "psql -d webodm_dev -c \"SET postgis.enable_outdb_rasters TO True;\""
sudo -u postgres bash -c "psql -d webodm_dev -c \"SET postgis.gdal_enabled_drivers TO 'GTiff';\""
# Add nginx
sudo apt-get install nginx -y
# Setup nodejs
curl -sL https://deb.nodesource.com/setup_6.x -o nodesource_setup.sh
sudo bash nodesource_setup.sh
sudo apt-get update
sudo apt-get install nodejs
sudo ln -s /usr/bin/nodejs /usr/bin/node
sudo npm install -g bower
# Setup virtualenv
virtualenv -p python3 env
. env/bin/activate
cd webodm/
pip install -r requirements.txt
# Build assets
sudo npm install -g webpack
npm install
webpack
# Configure other settings
cp webodm/settings.py webodm/local_settings.py
# Configure Docker (Processing Nodes)
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
sudo apt-add-repository 'deb https://apt.dockerproject.org/repo ubuntu-xenial main'
sudo apt-get update
sudo apt-get install -y docker-engine
sudo systemctl status docker
sudo usermod -aG docker $(whoami)

66
nginx/nginx.conf 100644
Wyświetl plik

@ -0,0 +1,66 @@
worker_processes 1;
# Change this if running outside docker!
user root root;
pid /tmp/nginx.pid;
error_log /tmp/nginx.error.log;
events {
worker_connections 1024; # increase if you have lots of clients
accept_mutex off; # set to 'on' if nginx worker_processes > 1
use epoll;
}
http {
include /etc/nginx/mime.types;
# fallback in case we can't determine a type
default_type application/octet-stream;
access_log /tmp/nginx.access.log combined;
sendfile on;
upstream app_server {
# fail_timeout=0 means we always retry an upstream even if it failed
# to return a good HTTP response
# for UNIX domain socket setups
server unix:/tmp/gunicorn.sock fail_timeout=0;
}
server {
listen 8000 deferred;
client_max_body_size 20G;
# set the correct host(s) for your site
server_name webodm.localhost;
keepalive_timeout 5;
# path for static files
location /static {
root /webodm/build;
}
location / {
# CORS settings
# These settings are VERY permissive, consider tightening them
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range';
add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range';
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# enable this if and only if you use HTTPS
# proxy_set_header X-Forwarded-Proto https;
proxy_set_header Host $http_host;
# we don't want nginx trying to do something clever with
# redirects, we set the Host: header above already.
proxy_redirect off;
proxy_pass http://app_server;
}
}
}

@ -1 +1 @@
Subproject commit 6ddbe1feba77fff0901a990f6a391bc48525ba74
Subproject commit bfc90c9cec21a06b88ed92f202a0a79901b8962d

Wyświetl plik

@ -2,16 +2,18 @@ anyjson==0.3.3
appdirs==1.4.0
APScheduler==3.2.0
coreapi==2.0.9
Django==1.10.5
Django==1.11.1
django-cors-headers==2.0.2
django-debug-toolbar==1.6
django-filter==0.15.3
django-filter==1.0.4
django-guardian==1.4.6
django-webpack-loader==0.3.3
djangorestframework==3.5.1
djangorestframework==3.6.3
djangorestframework-jwt==1.9.0
drf-nested-routers==0.11.1
funcsigs==1.0.2
futures==3.0.5
gunicorn==19.7.1
itypes==1.1.0
Markdown==2.6.7
openapi-codec==1.1.7
@ -21,7 +23,7 @@ pip-autoremove==0.9.0
psycopg2==2.6.2
PyJWT==1.4.2
pyparsing==2.1.10
pytz==2016.6.1
pytz==2017.2
requests==2.11.1
rfc3987==1.3.7
simplejson==3.10.0
@ -31,4 +33,3 @@ strict-rfc3339==0.7
tzlocal==1.3
uritemplate==3.0.0
webcolors==1.5
django-cors-headers==2.0.2

Wyświetl plik

@ -36,7 +36,7 @@ if [ $? -ne 0 ]; then
exit
fi
if [ $1 = "--setup-devenv" ] || [ $2 = "--setup-devenv" ]; then
if [ "$1" = "--setup-devenv" ] || [ "$2" = "--setup-devenv" ]; then
echo Setup git modules...
git submodule update --init
@ -58,7 +58,10 @@ fi
echo Running migrations
python manage.py migrate
if [[ $1 = "--create-default-pnode" ]]; then
echo Collecting static assets
python manage.py collectstatic --noinput
if [[ "$1" = "--create-default-pnode" ]]; then
echo "from nodeodm.models import ProcessingNode; ProcessingNode.objects.update_or_create(hostname='node-odm-1', defaults={'hostname': 'node-odm-1', 'port': 3000})" | python manage.py shell
fi
@ -73,7 +76,12 @@ echo Open a web browser and navigate to http://localhost:8000
echo -e "\033[39m"
echo -e "\033[91mNOTE:\033[39m Windows users using docker should replace localhost with the IP of their docker machine's IP. To find what that is, run: docker-machine ip") &
python manage.py runserver 0.0.0.0:8000
if [ "$1" = "--setup-devenv" ] || [ "$2" = "--setup-devenv" ] || [ "$1" = "--no-gunicorn" ]; then
python manage.py runserver 0.0.0.0:8000
else
nginx -c /webodm/nginx/nginx.conf
gunicorn webodm.wsgi --bind unix:/tmp/gunicorn.sock --preload
fi
# If this is executed, it means the previous command failed, don't display the congratulations message
kill %1

Wyświetl plik

@ -13,6 +13,8 @@ https://docs.djangoproject.com/en/1.10/ref/settings/
import os, sys
import datetime
import tzlocal
from django.contrib.messages import constants as messages
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@ -26,7 +28,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = 'gmarsutd!fee6_58=6k)2je#o2^&&)ovu1svjg8k^(a!7qa7r&'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TESTING = sys.argv[1:2] == ['test']
DEBUG = sys.argv[1:2] == ['runserver'] or TESTING
INTERNAL_IPS = ['127.0.0.1']
ALLOWED_HOSTS = ['*']
@ -42,6 +45,7 @@ INSTALLED_APPS = [
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.gis',
'django_filters',
'guardian',
'rest_framework',
'rest_framework_nested',
@ -132,7 +136,7 @@ AUTHENTICATION_BACKENDS = (
# https://docs.djangoproject.com/en/1.10/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = None # Use local server time
TIME_ZONE = tzlocal.get_localzone().zone
USE_I18N = True
USE_L10N = True
USE_TZ = True
@ -142,6 +146,7 @@ USE_TZ = True
# https://docs.djangoproject.com/en/1.10/howto/static-files/
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'build', 'static')
STATICFILES_DIRS = [
os.path.join(BASE_DIR, 'app', 'static'),
]
@ -239,7 +244,6 @@ JWT_AUTH = {
'JWT_EXPIRATION_DELTA': datetime.timedelta(hours=6),
}
TESTING = sys.argv[1:2] == ['test']
if TESTING:
MEDIA_ROOT = os.path.join(BASE_DIR, 'app', 'media_test')

Wyświetl plik

@ -8,9 +8,11 @@ https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/
"""
import os
from multiprocessing import Value
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "webodm.settings")
application = get_wsgi_application()
booted = Value('b', False)