From 6652c42d09e3f16ee6e7ced48b0043d73a740b2b Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 17 Feb 2017 20:43:58 -0500 Subject: [PATCH] Add Dockerfile, docker-compose.yml, S3 uploaded media settings (configurable via the environment), Docker instructions, and a .travis.yml to build/test the Docker image. --- .dockerignore | 4 + .gitignore | 1 - .travis.yml | 23 +++++ Dockerfile | 44 ++++++++++ Procfile | 2 +- app.json | 4 +- bakerydemo/settings/heroku.py | 22 ----- bakerydemo/settings/production.py | 68 +++++++++++++++ .../{heroku_wsgi.py => wsgi_production.py} | 0 docker-compose.yml | 33 ++++++++ docker-entrypoint.sh | 19 +++++ readme.md | 83 +++++++++++-------- requirements.txt | 2 +- requirements/{heroku.txt => production.txt} | 4 +- 14 files changed, 249 insertions(+), 60 deletions(-) create mode 100644 .dockerignore create mode 100644 .travis.yml create mode 100644 Dockerfile delete mode 100644 bakerydemo/settings/heroku.py create mode 100644 bakerydemo/settings/production.py rename bakerydemo/{heroku_wsgi.py => wsgi_production.py} (100%) create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh rename requirements/{heroku.txt => production.txt} (70%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9a71100 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +docker-compose.yml +Procfile +Vagrantfile diff --git a/.gitignore b/.gitignore index 7985bb1..489e2c2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,6 @@ bakerydemo/media/* bakerydemo/settings/local.py bakerydemodb __pycache__ -.* .vagrant/ /.vagrant/ /Vagrantfile.local diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e76f1c9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +sudo: required +services: + - docker + +env: + global: + - GREP_TIMEOUT=360 + +before_install: + - sudo apt-get update + - sudo apt-get install -qy -o Dpkg::Options::="--force-confold" docker-engine coreutils + +script: + # Bring up the postgres, redis, and app containers + - docker-compose up --build -d + + - timeout $GREP_TIMEOUT grep -m 1 'Running migrations' <(docker-compose logs --follow app 2>&1) + - timeout $GREP_TIMEOUT grep -m 1 'spawned uWSGI http 1' <(docker-compose logs --follow app 2>&1) + - docker-compose run app /venv/bin/python /code/manage.py check + +after_script: + - docker-compose logs + - docker images diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7ef77d9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.5-alpine + +ADD requirements/ /requirements/ +RUN set -ex \ + && apk add --no-cache --virtual .build-deps \ + gcc \ + g++ \ + make \ + libc-dev \ + musl-dev \ + linux-headers \ + pcre-dev \ + postgresql-dev \ + libjpeg-turbo-dev \ + && pyvenv /venv \ + && /venv/bin/pip install -U pip \ + && LIBRARY_PATH=/lib:/usr/lib /bin/sh -c "/venv/bin/pip install -r /requirements/production.txt" \ + && runDeps="$( \ + scanelf --needed --nobanner --recursive /venv \ + | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ + | sort -u \ + | xargs -r apk info --installed \ + | sort -u \ + )" \ + && apk add --virtual .python-rundeps $runDeps \ + && apk del .build-deps +RUN apk add --no-cache postgresql-client +RUN mkdir /code/ +WORKDIR /code/ +ADD . /code/ +EXPOSE 8000 + +# Add custom environment variables needed by Django or your settings file here: +ENV DJANGO_SETTINGS_MODULE=bakerydemo.settings.production DJANGO_DEBUG=off + +# uWSGI configuration (customize as needed): +ENV UWSGI_VIRTUALENV=/venv UWSGI_WSGI_FILE=bakerydemo/wsgi_production.py UWSGI_HTTP=:8000 UWSGI_MASTER=1 UWSGI_WORKERS=2 UWSGI_THREADS=8 UWSGI_UID=1000 UWSGI_GID=2000 + +# Call collectstatic with dummy environment variables: +RUN DATABASE_URL=postgres://none REDIS_URL=none /venv/bin/python manage.py collectstatic --noinput + +# start uWSGI, using a wrapper script to allow us to easily add more commands to container startup: +ENTRYPOINT ["/code/docker-entrypoint.sh"] +CMD ["/venv/bin/uwsgi", "--http-auto-chunked", "--http-keepalive"] diff --git a/Procfile b/Procfile index 541c935..688af51 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ release: yes "yes" | python manage.py migrate -web: gunicorn bakerydemo.heroku_wsgi --log-file - +web: uwsgi --http-socket=:$PORT --master --workers=2 --threads=8 --die-on-term --wsgi-file=bakerydemo/wsgi_production.py diff --git a/app.json b/app.json index 7c22067..78e9405 100644 --- a/app.json +++ b/app.json @@ -4,7 +4,9 @@ "repository": "https://github.com/wagtail/bakerydemo", "keywords": ["wagtail", "django"], "env": { - "DJANGO_SETTINGS_MODULE": "bakerydemo.settings.heroku" + "DJANGO_DEBUG": "off", + "DJANGO_SETTINGS_MODULE": "bakerydemo.settings.production", + "DJANGO_SECURE_SSL_REDIRECT": "on" }, "scripts": { "postdeploy": "django-admin.py migrate && django-admin.py load_initial_data && echo 'from wagtail.wagtailimages.models import Rendition; Rendition.objects.all().delete()' | django-admin.py shell" diff --git a/bakerydemo/settings/heroku.py b/bakerydemo/settings/heroku.py deleted file mode 100644 index d67157b..0000000 --- a/bakerydemo/settings/heroku.py +++ /dev/null @@ -1,22 +0,0 @@ -import dj_database_url - -from .base import * - - -# Accept all hostnames, since we don't know in advance which hostname will be used for any given Heroku instance. -# IMPORTANT: Set this to a real hostname when using this in production! -# See https://docs.djangoproject.com/en/1.10/ref/settings/#allowed-hosts -ALLOWED_HOSTS = ['*', ] - -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' - -# BASE_URL required for notification emails -BASE_URL = 'http://localhost:8000' - -db_from_env = dj_database_url.config(conn_max_age=500) -DATABASES['default'].update(db_from_env) - -# Simplified static file serving. -# https://warehouse.python.org/project/whitenoise/ - -STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage' diff --git a/bakerydemo/settings/production.py b/bakerydemo/settings/production.py new file mode 100644 index 0000000..0249cbd --- /dev/null +++ b/bakerydemo/settings/production.py @@ -0,0 +1,68 @@ +import os +import dj_database_url +import random +import string + +from .base import * + +DEBUG = os.getenv('DJANGO_DEBUG', 'off') == 'on' + +# DJANGO_SECRET_KEY *should* be specified in the environment. If it's not, generate an ephemeral key. +if 'DJANGO_SECRET_KEY' in os.environ: + SECRET_KEY = os.environ['DJANGO_SECRET_KEY'] +else: + # Use if/else rather than a default value to avoid calculating this if we don't need it + print("WARNING: DJANGO_SECRET_KEY not found in os.environ. Generating ephemeral SECRET_KEY.") + SECRET_KEY = ''.join([random.SystemRandom().choice(string.printable) for i in range(50)]) + +# Make sure Django can detect a secure connection properly on Heroku: +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + +# Redirect all requests to HTTPS +SECURE_SSL_REDIRECT = os.getenv('DJANGO_SECURE_SSL_REDIRECT', 'off') == 'on' + +# Accept all hostnames, since we don't know in advance which hostname will be used for any given Heroku instance. +# IMPORTANT: Set this to a real hostname when using this in production! +# See https://docs.djangoproject.com/en/1.10/ref/settings/#allowed-hosts +ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '*').split(';') + +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + +# BASE_URL required for notification emails +BASE_URL = 'http://localhost:8000' + +db_from_env = dj_database_url.config(conn_max_age=500) +DATABASES['default'].update(db_from_env) + +# Simplified static file serving. +# https://warehouse.python.org/project/whitenoise/ + +MIDDLEWARE.append('whitenoise.middleware.WhiteNoiseMiddleware') +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' + +if 'AWS_STORAGE_BUCKET_NAME' in os.environ: + AWS_STORAGE_BUCKET_NAME = os.getenv('AWS_STORAGE_BUCKET_NAME') + AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID', '') + AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY', '') + AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME + AWS_AUTO_CREATE_BUCKET = True + + INSTALLED_APPS.append('storages') + MEDIA_URL = "https://%s/" % AWS_S3_CUSTOM_DOMAIN + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + }, + }, +} diff --git a/bakerydemo/heroku_wsgi.py b/bakerydemo/wsgi_production.py similarity index 100% rename from bakerydemo/heroku_wsgi.py rename to bakerydemo/wsgi_production.py diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a1e5e73 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +version: '2' + +services: + db: + environment: + POSTGRES_DB: app_db + POSTGRES_USER: app_user + POSTGRES_PASSWORD: changeme + restart: always + image: postgres:9.6 + expose: + - "5432" + redis: + restart: always + image: redis:3.0 + expose: + - "6379" + app: + environment: + DJANGO_SECRET_KEY: changeme + DATABASE_URL: postgres://app_user:changeme@db/app_db + REDIS_URL: redis://redis + build: + context: . + dockerfile: ./Dockerfile + links: + - db:db + - redis:redis + ports: + - "8000:8000" + depends_on: + - db + - redis diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..971fd8b --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +until psql $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - continuing" + +if [ "$1" = '/venv/bin/uwsgi' ]; then + /venv/bin/python manage.py migrate --noinput +fi + +if [ "x$DJANGO_LOAD_INITIAL_DATA" = 'xon' ]; then + /venv/bin/python manage.py load_initial_data +fi + +exec "$@" diff --git a/readme.md b/readme.md index 8d4ce09..64ac578 100644 --- a/readme.md +++ b/readme.md @@ -26,7 +26,7 @@ Run the following commands: ```bash git clone git@github.com:wagtail/bakerydemo.git -cd wagtaildemo +cd bakerydemo vagrant up vagrant ssh # then, within the SSH session: @@ -38,9 +38,39 @@ interface at [http://localhost:8000/admin/](http://localhost:8000/admin/). Log into the admin with the credentials ``admin / changeme``. -Setup without Vagrant ------ -Don't want to set up a whole VM to try out Wagtail? No problem. +Setup with Docker +----------------- + +### Dependencies +* [Docker](https://docs.docker.com/engine/installation/) + +### Installation +Run the following commands: + +```bash +git clone git@github.com:wagtail/bakerydemo.git +cd bakerydemo +docker-compose up --build -d +docker-compose run app /venv/bin/python manage.py load_initial_data +``` + +The demo site will now be accessible at [http://localhost:8000/](http://localhost:8000/) and the Wagtail admin +interface at [http://localhost:8000/admin/](http://localhost:8000/admin/). + +Log into the admin with the credentials ``admin / changeme``. + +**Important:** This `docker-compose.yml` is configured for local testing only, and is not intended for production use. + +### Debugging +To tail the logs from the Docker containers in realtime, run: + +```bash +docker-compose logs -f +``` + +Local Setup +----------- +Don't want to set up a whole VM nor use Docker to try out Wagtail? No problem. ### Dependencies * [PIP](https://github.com/pypa/pip) @@ -89,53 +119,40 @@ update in the browser. Once finished, click `View` to see the public site. Log into the admin with the credentials ``admin / changeme``. +To prevent the demo site from regenerating a new Django `SECRET_KEY` each time Heroku restarts your site, you should set +a `DJANGO_SECRET_KEY` environment variable in Heroku using the web interace or the [CLI](https://devcenter.heroku.com/articles/heroku-cli). If using the CLI, you can set a `SECRET_KEY` like so: + + heroku config:set DJANGO_SECRET_KEY=changeme + To learn more about Heroku, read [Deploying Python and Django Apps on Heroku](https://devcenter.heroku.com/articles/deploying-python). ### Storing Wagtail Media Files on AWS S3 -If you have deployed the demo site to Heroku, you may want to perform some additional setup. Heroku uses an -[ephemeral filesystem](https://devcenter.heroku.com/articles/dynos#ephemeral-filesystem). In laymen's terms, this means -that uploaded images will disappear at a minimum of once per day, and on each application deployment. To mitigate this, -you can host your media on S3. +If you have deployed the demo site to Heroku or via Docker, you may want to perform some additional setup. Heroku uses an +[ephemeral filesystem](https://devcenter.heroku.com/articles/dynos#ephemeral-filesystem), and Docker-based hosting +environments typically work in the same manner. In laymen's terms, this means that uploaded images will disappear at a +minimum of once per day, and on each application deployment. To mitigate this, you can host your media on S3. This documentation assumes that you have an AWS account, an IAM user, and a properly configured S3 bucket. These topics are outside of the scope of this documentation; the following [blog post](https://wagtail.io/blog/amazon-s3-for-media-files/) will walk you through those steps. -Next, you will need to add `django-storages` and `boto3` to `requirements/heroku.txt`. - -Then you will need to edit `settings/heroku.py`: - - INSTALLED_APPS.append('storages') - -You will also need to add the S3 bucket and access credentials for the IAM user that you created. - - AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME', 'changeme') - AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', 'changeme') - AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', 'changeme') - AWS_S3_CUSTOM_DOMAIN = '{}.s3.amazonaws.com'.format(AWS_STORAGE_BUCKET_NAME) - -Next, you will need to set these values in the Heroku environment. The next steps assume that you have -the [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) installed and configured. You will -execute the following commands to set the aforementioned environment variables: +This demo site comes preconfigured with a production settings file that will enable S3 for uploaded media storage if +``AWS_STORAGE_BUCKET_NAME`` is defined in the shell environment. All you need to do is set the following environment +variables. If using Heroku, you will first need to install and configure the [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli). Then, execute the following commands to set the aforementioned environment variables: heroku config:set AWS_STORAGE_BUCKET_NAME=changeme heroku config:set AWS_ACCESS_KEY_ID=changeme heroku config:set AWS_SECRET_ACCESS_KEY=changeme -Do not forget to replace the `changeme` with the actual values for your AWS account. +Do not forget to replace the `changeme` with the actual values for your AWS account. If you're using a different hosting +environment, set the same environment variables there using the method appropriate for your environment. -Finally, we need to configure the `MEDIA_URL` as well as inform Django that we want to use `boto3` for the storage -backend: - - MEDIA_URL = 'https://{}/'.format(AWS_S3_CUSTOM_DOMAIN)' - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' - -Commit these changes and push to Heroku and you should now have persistent media storage! +Once Heroku restarts your application or your Docker container is refreshed, you should have persistent media storage! ### Sending email from the contact form -The following setting in `base.py` and `heroku.py` ensures that live email is not sent by the demo contact form. +The following setting in `base.py` and `production.py` ensures that live email is not sent by the demo contact form. `EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'` diff --git a/requirements.txt b/requirements.txt index 56e6917..ea77c2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ --r requirements/heroku.txt +-r requirements/production.txt diff --git a/requirements/heroku.txt b/requirements/production.txt similarity index 70% rename from requirements/heroku.txt rename to requirements/production.txt index 91e2004..48bbbb2 100644 --- a/requirements/heroku.txt +++ b/requirements/production.txt @@ -1,6 +1,8 @@ -r base.txt # Additional dependencies for Heroku deployment dj-database-url==0.4.1 -gunicorn==19.6.0 +uwsgi==2.0.14 psycopg2==2.6.2 whitenoise==3.2.2 +boto==2.45.0 +django-storages==1.5.2