Merge pull request #30 from swiftugandan/deploy-using-docker-compose

Deploy using docker compose

.gitignore

@@ -0,0 +1,12 @@
+media_files/encoded/
+media_files/original/
+postgres_data/
+celerybeat-schedule
+logs/
+pids/
+static/admin/
+static/ckeditor/
+static/debug_toolbar/
+static/mptt/
+static/rest_framework/
+cms/local_settings.py

.mailmap

@@ -0,0 +1 @@
+Swift Ugandan <swiftugandan@gmail.com> <swiftugandan@gmail.com>

Dockerfile

@@ -0,0 +1,64 @@
+FROM python:3.8-buster AS compile-image
+
+SHELL ["/bin/bash", "-c"]
+
+# Set up virtualenv
+ENV VIRTUAL_ENV=/home/mediacms.io
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PIP_NO_CACHE_DIR=1
+
+RUN mkdir -p /home/mediacms.io/mediacms/{logs,pids} && cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV
+
+# Install dependencies:
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . /home/mediacms.io/mediacms
+WORKDIR /home/mediacms.io/mediacms
+
+RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip && \
+    mkdir -p /home/mediacms.io/mediacms/media_files/hls Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/ && \
+    unzip -j Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/mp4hls -d Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/ && \
+    rm Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip
+
+############ RUNTIME IMAGE ############
+FROM python:3.8-slim-buster as runtime-image
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV ADMIN_USER='admin'
+ENV ADMIN_PASSWORD='mediacms'
+ENV ADMIN_EMAIL='admin@localhost'
+
+# See: https://github.com/celery/celery/issues/6285#issuecomment-715316219
+ENV CELERY_APP='cms'
+
+# Use these to toggle which processes supervisord should run
+ENV ENABLE_UWSGI='yes'
+ENV ENABLE_NGINX='yes'
+ENV ENABLE_CELERY_BEAT='yes'
+ENV ENABLE_CELERY_SHORT='yes'
+ENV ENABLE_CELERY_LONG='yes'
+ENV ENABLE_MIGRATIONS='yes'
+
+# Set up virtualenv
+ENV VIRTUAL_ENV=/home/mediacms.io
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+COPY --from=compile-image /home/mediacms.io /home/mediacms.io
+
+RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends \
+    supervisor nginx ffmpeg imagemagick procps -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get purge --auto-remove && \
+    apt-get clean
+
+WORKDIR /home/mediacms.io/mediacms
+
+EXPOSE 9000 80
+
+RUN chmod +x ./deploy/docker/entrypoint.sh
+
+ENTRYPOINT ["./deploy/docker/entrypoint.sh"]
+
+CMD ["./deploy/docker/start.sh"]

cms/settings.py

@@ -419,6 +419,8 @@ CELERY_BEAT_SCHEDULE = {
 # TODO: beat, delete chunks from media root
 # chunks_dir after xx days...(also uploads_dir)
 
+LOCAL_INSTALL = False
+
 try:
     # keep a local_settings.py file for local overrides
     from .local_settings import *
@@ -434,4 +436,7 @@ if "http" not in FRONTEND_HOST:
     # FRONTEND_HOST needs a http:// preffix
     FRONTEND_HOST = f"http://{FRONTEND_HOST}"
 
-SSL_FRONTEND_HOST = FRONTEND_HOST.replace("http", "https")
+if LOCAL_INSTALL:
+    SSL_FRONTEND_HOST = FRONTEND_HOST.replace("http", "https")
+else:
+    SSL_FRONTEND_HOST = FRONTEND_HOST

deploy/docker/README.md

@@ -0,0 +1,3 @@
+# MediaCMS on Docker
+
+See: [Details](../../docs/Docker_deployment.md)

deploy/docker/entrypoint.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+# forward request and error logs to docker log collector
+ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log && \
+ln -sf /dev/stdout /var/log/nginx/mediacms.io.access.log && ln -sf /dev/stderr /var/log/nginx/mediacms.io.error.log
+cp /home/mediacms.io/mediacms/deploy/docker/local_settings.py /home/mediacms.io/mediacms/cms/local_settings.py
+
+mkdir -p /home/mediacms.io/mediacms/{logs,pids}
+touch /home/mediacms.io/mediacms/logs/debug.log
+
+chown -R www-data. /home/mediacms.io/
+
+TARGET_GID=$(stat -c "%g" /home/mediacms.io/mediacms/)
+
+EXISTS=$(cat /etc/group | grep $TARGET_GID | wc -l)
+
+# Create new group using target GID and add www-data user
+if [ $EXISTS == "0" ]; then
+    groupadd -g $TARGET_GID tempgroup
+    usermod -a -G tempgroup www-data
+else
+    # GID exists, find group name and add
+    GROUP=$(getent group $TARGET_GID | cut -d: -f1)
+    usermod -a -G $GROUP www-data
+fi
+
+chmod +x /home/mediacms.io/mediacms/deploy/docker/start.sh /home/mediacms.io/mediacms/deploy/docker/prestart.sh
+
+exec "$@"

deploy/docker/local_settings.py

@@ -0,0 +1,32 @@
+FRONTEND_HOST = 'http://localhost'
+PORTAL_NAME = 'MediaCMS'
+SECRET_KEY = 'ma!s3^b-cw!f#7s6s0m3*jx77a@riw(7701**(r=ww%w!2+yk2'
+POSTGRES_HOST = 'db'
+REDIS_LOCATION = "redis://redis:6379/1"
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": "mediacms",
+        "HOST": POSTGRES_HOST,
+        "PORT": "5432",
+        "USER": "mediacms",
+        "PASSWORD": "mediacms",
+    }
+}
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_LOCATION,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    }
+}
+
+# CELERY STUFF
+BROKER_URL = REDIS_LOCATION
+CELERY_RESULT_BACKEND = BROKER_URL
+
+DEBUG = False

deploy/docker/nginx_http_only.conf

@@ -0,0 +1,30 @@
+server {
+    listen 80 ;
+
+    gzip on;
+    access_log /var/log/nginx/mediacms.io.access.log;
+
+    error_log  /var/log/nginx/mediacms.io.error.log  warn;
+
+    location /static {
+        alias /home/mediacms.io/mediacms/static ;
+    }
+
+    location /media/original {
+        alias /home/mediacms.io/mediacms/media_files/original;
+    }
+
+    location /media {
+        alias /home/mediacms.io/mediacms/media_files ;
+    }
+
+    location / {
+        add_header 'Access-Control-Allow-Origin' '*';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+        add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
+
+        include /etc/nginx/sites-enabled/uwsgi_params;
+        uwsgi_pass 127.0.0.1:9000;
+    }
+}

deploy/docker/prestart.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+RANDOM_ADMIN_PASS=`python -c "import secrets;chars = 'abcdefghijklmnopqrstuvwxyz0123456789';print(''.join(secrets.choice(chars) for i in range(10)))"`
+ADMIN_PASSWORD=${ADMIN_PASSWORD:-$RANDOM_ADMIN_PASS}
+
+if [ X"$ENABLE_MIGRATIONS" = X"yes" ]; then
+    python manage.py migrate
+    python manage.py loaddata fixtures/encoding_profiles.json
+    python manage.py loaddata fixtures/categories.json
+    python manage.py collectstatic --noinput
+
+    echo "Admin Password: $ADMIN_PASSWORD"
+
+    # post_save, needs redis to succeed (ie. migrate depends on redis)
+    DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
+        --no-input \
+        --username=$ADMIN_USER \
+        --email=$ADMIN_EMAIL \
+        --database=default || true
+
+    # echo "Updating hostname ..."
+    # TODO: Get the FRONTEND_HOST from cms/local_settings.py
+    # echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell
+fi
+
+# Setting up internal nginx server
+# HTTPS setup is delegated to a reverse proxy running infront of the application
+
+cp deploy/docker/nginx_http_only.conf /etc/nginx/sites-available/default
+cp deploy/docker/nginx_http_only.conf /etc/nginx/sites-enabled/default
+cp deploy/docker/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
+cp deploy/docker/nginx.conf /etc/nginx/
+
+#### Supervisord Configurations #####
+
+cp deploy/docker/supervisord/supervisord-debian.conf /etc/supervisor/conf.d/supervisord-debian.conf
+
+if [ X"$ENABLE_UWSGI" = X"yes" ] ; then
+    echo "Enabling uwsgi app server"
+    cp deploy/docker/supervisord/supervisord-uwsgi.conf /etc/supervisor/conf.d/supervisord-uwsgi.conf
+fi
+
+if [ X"$ENABLE_NGINX" = X"yes" ] ; then
+    echo "Enabling nginx as uwsgi app proxy and media server"
+    cp deploy/docker/supervisord/supervisord-nginx.conf /etc/supervisor/conf.d/supervisord-nginx.conf
+fi
+
+if [ X"$ENABLE_CELERY_BEAT" = X"yes" ] ; then
+    echo "Enabling celery-beat scheduling server"
+    cp deploy/docker/supervisord/supervisord-celery_beat.conf /etc/supervisor/conf.d/supervisord-celery_beat.conf
+fi
+
+if [ X"$ENABLE_CELERY_SHORT" = X"yes" ] ; then
+    echo "Enabling celery-short task worker"
+    cp deploy/docker/supervisord/supervisord-celery_short.conf /etc/supervisor/conf.d/supervisord-celery_short.conf
+fi
+
+if [ X"$ENABLE_CELERY_LONG" = X"yes" ] ; then
+    echo "Enabling celery-long task worker"
+    cp deploy/docker/supervisord/supervisord-celery_long.conf /etc/supervisor/conf.d/supervisord-celery_long.conf
+fi

deploy/docker/reverse_proxy/certs/localhost.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICwzCCAaugAwIBAgIJAOyvdwguJQd+MA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV
+BAMTCWxvY2FsaG9zdDAeFw0yMTAxMjQxMjUwMzFaFw0zMTAxMjIxMjUwMzFaMBQx
+EjAQBgNVBAMTCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAONswEwBzkgoO+lkewiKUnwvYqC54qleCUg9hidqjoyzd5XWKh1mIF7aaSCG
+rJGSxCce8CbqAqGkpvsgXzwwbY72l7FwmAXFHO5ObQfpmFhjt2fsKRM9MTCo/UyU
+liuhgP+Q+BNzUontTUC40NVHs8R7IHG4z8unB7qB/7zGK2tfilLB8JDqPTkc22vN
+C4P1YxiGyY5bm37wQrroC9zPJ8bqanrF9Y90QJHubibnPWqnZvK2HkDWjp5LYkn8
+IuzBycs1cLd8eMjU9aT72kweykvnGDDc3YbXFzT2zBTGSFEBROsVdPrNF9PaeE3j
+pu4UZ8Ge3Fp3VYd+04DnWtbQq0MCAwEAAaMYMBYwFAYDVR0RBA0wC4IJbG9jYWxo
+b3N0MA0GCSqGSIb3DQEBBQUAA4IBAQAdm2aGn4evosbdWgBHgzr6oYWBIiPpf1SA
+GXizuf5OaMActFP0rZ0mogndLH5d51J2qqSfOtaWSA5qwlPvDSTn1nvJeHoVLfZf
+kQHaB7/DaOPGsZCQBELPhYHwl7+Ej3HYE+siiaRfjC2NVgf8P/pAsTlKbe2e+34l
+GwWSFol24w5xAmUezCF41JiZbqHoZhSh7s/PuJnK2RvhpjkrIot8GvxnbvOcKDIv
+JzEKo3qPq8pc5RBkpP7Kp2+EgAYn1xAn0CekxZracW/MY+tg2mCeFucZW2V1iwVs
+LpAw6GJnjYz5mbrQskPbrJ9t78JGUKQ0kL/VUTfryUHMHYCiJlvd
+-----END CERTIFICATE-----

deploy/docker/reverse_proxy/certs/localhost.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA42zATAHOSCg76WR7CIpSfC9ioLniqV4JSD2GJ2qOjLN3ldYq
+HWYgXtppIIaskZLEJx7wJuoCoaSm+yBfPDBtjvaXsXCYBcUc7k5tB+mYWGO3Z+wp
+Ez0xMKj9TJSWK6GA/5D4E3NSie1NQLjQ1UezxHsgcbjPy6cHuoH/vMYra1+KUsHw
+kOo9ORzba80Lg/VjGIbJjlubfvBCuugL3M8nxupqesX1j3RAke5uJuc9aqdm8rYe
+QNaOnktiSfwi7MHJyzVwt3x4yNT1pPvaTB7KS+cYMNzdhtcXNPbMFMZIUQFE6xV0
++s0X09p4TeOm7hRnwZ7cWndVh37TgOda1tCrQwIDAQABAoIBAQCmKKyOW7tlCNBN
+AzbI1JbTWKOMnoM2DxhlCV5cqgOgVPcIKEL428bGxniMZRjr+vkJRBddtxdZFj1R
+uSMbjJ5fF1dZMtQ/UvaCPhZ283p1CdXUPbz863ZnAPCf5Oea1RK0piw5ucYSM6h/
+owgg65Qx92uK6uYW+uAwqg440+ihNvnaZoVTx5CjZbL9KISkrlNJnuYiB5vzOD0i
+UVklO5Qz8VCuOcOVGZCA2SxHm4HAbg/aiQnpaUa9de4TsZ4ygF66pZh77T0wNOos
+sS1riKtHQpX+osJyoTI/rIKFAhycsZ+AA7Qpu6GW4xQlNS6K8vRiIbktwkC+IT0O
+RSn8Dg7BAoGBAPe5R8SpgXx9jKdA1eFa/Vjx5bmB96r2MviIOIWF8rs2K33xe+rj
+v+BZ2ZjdpVjcm2nRMf9r/eDq2ScNFWmKoZsUmdyT84Qq9yLcTSUdno+zCy+L0LNH
+DqJq5jIxJaV7amHeR/w10BVuiDmzhSsTmhfnXTUGRO/h2PjRyC3yEYdxAoGBAOsF
+2+gTsdOGlq6AVzW5MLZkreq8WCU2wWpZRiCPh6HJa8htuynYxO5AWUiNUbYKddj2
+0za9DFiXgH+Oo8wrkTYLEdN0T5/o+ScL5t3VG3m9R6pnuudLC2vmGQP0hNuZUpnF
+7FzdJ85h6taR2bM1zFzOfl81K0BhTHGxTU2r70vzAoGAVXuLJ3LyqtnMKn72DzDN
+0d6PTkdqBoW0qwyerHy/eRjFQ02MXE7BDJMUwmphv1tJCefVX/WNAwsnahFavTPI
+dnJSccpgMtB8vXvV5yPkbmPzTTHrD6JKi4Nl8hYBjqwa1rDUmFSdfHfK7FZlcqrt
+9qexAzYpnbmKnLoPYMNyhxECgYEAm5OCUeuPoL2MS7GLiXWwyFx3QFczZlcLzBGS
+uYUpvLBwF/qDlhz3p9uS/tMFzyK3hktF4Ate+9o2ZroOtd31PzgusbJh7zIylGVt
+i1VB3eGtaiFGeUuVIPTthE++Dvw80KxTXdnMOvNYmHduDBLF2H2c6/tvSSvfhbdf
+u9XgD38CgYAiLcVySxMKNpsXatuC31wjT+rnaH22SD/7pXe2q6MRW/s+bGOspu0v
+NeJSLoM98v8F99q0W0lgqesYJVI20Frru0DfXIp60ryaDolzve3Iwk8SOJUlcnUG
+cCtmPUkjyr18QAlrcCB4PozJGjpPWyabaY8gGwo8wAEpJWHrIJlHew==
+-----END RSA PRIVATE KEY-----

deploy/docker/reverse_proxy/client_max_body_size.conf

@@ -0,0 +1 @@
+client_max_body_size 1g;

deploy/docker/start.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env sh
+set -e
+
+# If there's a prestart.sh script in the /app directory, run it before starting
+PRE_START_PATH=deploy/docker/prestart.sh
+echo "Checking for script in $PRE_START_PATH"
+if [ -f $PRE_START_PATH ] ; then
+    echo "Running script $PRE_START_PATH"
+    . $PRE_START_PATH
+else
+    echo "There is no script $PRE_START_PATH"
+fi
+
+# Start Supervisor, with Nginx and uWSGI
+echo "Starting server using supervisord..."
+
+exec /usr/bin/supervisord

deploy/docker/supervisord/supervisord-celery_beat.conf

@@ -0,0 +1,12 @@
+[program:celery_beat]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=0
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=300
+startinorder=true
+command=/home/mediacms.io/bin/celery beat --pidfile=/home/mediacms.io/mediacms/pids/beat%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_beat.log

deploy/docker/supervisord/supervisord-celery_long.conf

@@ -0,0 +1,13 @@
+[program:celery_long]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=10
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=500
+startinorder=true
+startsecs=0
+command=/home/mediacms.io/bin/celery multi start long1 --pidfile=/home/mediacms.io/mediacms/pids/%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_long.log -Ofair --prefetch-multiplier=1 -Q long_tasks

deploy/docker/supervisord/supervisord-celery_short.conf

@@ -0,0 +1,12 @@
+[program:celery_short]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=0
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=400
+startinorder=true
+command=/home/mediacms.io/bin/celery multi start short1 short2 --pidfile=/home/mediacms.io/mediacms/pids/%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_short.log --soft-time-limit=300 -c10 -Q short_tasks

deploy/docker/supervisord/supervisord-debian.conf

@@ -0,0 +1,2 @@
+[supervisord]
+nodaemon=true

deploy/docker/supervisord/supervisord-nginx.conf

@@ -0,0 +1,11 @@
+[program:nginx]
+command=/usr/sbin/nginx -g 'daemon off;'
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=200
+startinorder=true
+startsecs=0
+# Graceful stop, see http://nginx.org/en/docs/control.html
+stopsignal=QUIT

deploy/docker/supervisord/supervisord-uwsgi.conf

@@ -0,0 +1,9 @@
+[program:uwsgi]
+command=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/deploy/docker/uwsgi.ini
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=100
+startinorder=true
+startsecs=0

deploy/docker/uwsgi.ini

@@ -0,0 +1,23 @@
+[uwsgi]
+
+chdir = /home/mediacms.io/mediacms/
+virtualenv = /home/mediacms.io
+module = cms.wsgi
+
+uid=www-data
+gid=www-data
+
+processes = 2
+threads = 2
+
+master = true
+
+socket = 127.0.0.1:9000
+
+workers = 2
+
+vacuum = true
+
+hook-master-start = unix_signal:15 gracefully_kill_them_all
+need-app = true
+die-on-term = true

deploy/local_install/mediacms.io

@@ -59,7 +59,7 @@ server {
     location /media/original {
         alias /home/mediacms.io/mediacms/media_files/original;
         #auth_basic "auth protected area";
-        #auth_basic_user_file /home/mediacms.io/mediacms/deploy/.htpasswd;
+        #auth_basic_user_file /home/mediacms.io/mediacms/deploy/local_install/.htpasswd;
     }
 
     location /media {

deploy/local_install/mediacms.service

@@ -2,7 +2,7 @@
 Description=MediaCMS uwsgi
 
 [Service]
-ExecStart=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/uwsgi.ini
+ExecStart=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/deploy/local_install/uwsgi.ini
 ExecStop=/usr/bin/killall -9 uwsgi
 RestartSec=3
 #ExecRestart=killall -9 uwsgi; sleep 5; /home/sss/bin/uwsgi --ini /home/sss/wordgames/uwsgi.ini

deploy/local_install/nginx.conf

@@ -0,0 +1,41 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 10240;
+}
+
+    worker_rlimit_nofile    20000;  #each connection needs a filehandle (or 2 if you are proxying)
+http {
+    proxy_connect_timeout 75;
+    proxy_read_timeout 12000;
+    client_max_body_size 5800M;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 10;
+    types_hash_max_size 2048;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+    ssl_prefer_server_ciphers on;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    gzip on;
+    gzip_disable "msie6";
+
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                           '"$request" $status $body_bytes_sent '
+                           '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+    include /etc/nginx/conf.d/*.conf;
+    include /etc/nginx/sites-enabled/*;
+}
+

deploy/local_install/uwsgi.ini

@@ -1,8 +1,8 @@
 [uwsgi]
 
-chdir           = /home/mediacms.io/mediacms/
+chdir = /home/mediacms.io/mediacms/
 virtualenv = /home/mediacms.io
-module          = cms.wsgi
+module = cms.wsgi
 
 uid=www-data
 gid=www-data
@@ -10,7 +10,7 @@ gid=www-data
 processes = 2
 threads = 2
 
-master          = true
+master = true
 
 socket = 127.0.0.1:9000
 #socket = /home/mediacms.io/mediacms/deploy/uwsgi.sock
@@ -19,7 +19,7 @@ socket = 127.0.0.1:9000
 workers = 2
 
 
-vacuum          = true
+vacuum = true
 
 logto = /home/mediacms.io/mediacms/logs/errorlog.txt
 

deploy/local_install/uwsgi_params

@@ -0,0 +1,16 @@
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;

deploy/uwsgi.ini

@@ -1,19 +0,0 @@
-[uwsgi]
-
-chdir = /home/mediacms.io/mediacms/
-virtualenv = /home/mediacms.io
-module = cms.wsgi
-
-uid = www-data
-gid = www-data
-
-processes = 10
-threads = 10
-master = true
-workers = 8
-vacuum = true
-
-socket = 127.0.0.1:9000
-
-logto = /home/mediacms.io/mediacms/logs/errorlog.txt
-

docker-compose-http-proxy.yaml

@@ -0,0 +1,89 @@
+version: "3"
+
+services:
+  nginx-proxy:
+    image: jwilder/nginx-proxy
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - ./deploy/docker/reverse_proxy/client_max_body_size.conf:/etc/nginx/conf.d/client_max_body_size.conf:ro
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+      VIRTUAL_HOST: localhost
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - ../postgres_data/:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

docker-compose-https-proxy.yaml

@@ -0,0 +1,91 @@
+version: "3"
+
+services:
+  nginx-proxy:
+    image: jwilder/nginx-proxy
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - ./deploy/docker/reverse_proxy/certs/:/etc/nginx/certs/
+      - ./deploy/docker/reverse_proxy/client_max_body_size.conf:/etc/nginx/conf.d/client_max_body_size.conf:ro
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+      VIRTUAL_HOST: localhost
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 2
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - ../postgres_data/:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

docker-compose-named-volumes.yaml

@@ -0,0 +1,91 @@
+version: "3"
+
+services:
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/deploy/docker/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    ports:
+      - "80:80"
+    volumes:
+      - media_store:/home/mediacms.io/mediacms/media_files/
+      - static_store:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/deploy/docker/local_settings.py
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/deploy/docker/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - media_store:/home/mediacms.io/mediacms/media_files/
+      - static_store:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/deploy/docker/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+volumes:
+  postgres_data:
+  media_store:
+  static_store:

docker-compose.yaml

@@ -0,0 +1,83 @@
+version: "3"
+
+services:
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    ports:
+      - "80:80"
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./:/home/mediacms.io/mediacms/
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - ../postgres_data:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

docs/Docker_deployment.md

@@ -0,0 +1,40 @@
+# MediaCMS on Docker
+
+The mediacms image is built to use supervisord as the main process, which manages one or more services required to run mediacms. We can toggle which services are run in a given container by setting the environment variables below to `yes` or `no`:
+
+* ENABLE_UWSGI
+* ENABLE_NGINX
+* ENABLE_CELERY_BEAT
+* ENABLE_CELERY_SHORT
+* ENABLE_CELERY_LONG
+* ENABLE_MIGRATIONS
+
+By default, all these services are enabled, but in order to create a scaleable deployment, some of them can be disabled, splitting the service up into smaller services.
+
+Also see the `Dockerfile` for other environment variables which you may wish to override. Application settings, eg. `FRONTEND_HOST` can also be overridden by updating the `deploy/docker/local_settings.py` file.
+
+See example deployments in the sections below. These example deployments have been tested on `docker-compose version 1.27.4` running on `Docker version 19.03.13`
+
+To run, update the configs above if necessary, build the image by running `docker-compose build`, then run `docker-compose run`
+
+## Simple Deployment, accessed as http://localhost
+
+The main container runs migrations, mediacms_web, celery_beat, celery_workers (celery_short and celery_long services), exposed on port 80 supported by redis and postgres database. The FRONTEND_HOST in `deploy/docker/local_settings.py` is configured as http://localhost, on the docker host machine.
+
+## Advanced Deployment, accessed as http://localhost:8000
+
+Here we can run 1 mediacms_web instance, with the FRONTEND_HOST in `deploy/docker/local_settings.py` configured as http://localhost:8000. This is bootstrapped by a single migrations instance and supported by a single celery_beat instance and 1 or more celery_worker instances. Redis and postgres containers are also used for persistence. Clients can access the service on http://localhost:8000, on the docker host machine. This is similar to [this deployment](../docker-compose.yaml), with a `port` defined in FRONTEND_HOST.
+
+## Advanced Deployment, with reverse proxy, accessed as http://mediacms.io
+
+Here we can use `jwilder/nginx-proxy` to reverse proxy to 1 or more instances of mediacms_web supported by other services as mentioned in the previous deployment. The FRONTEND_HOST in `deploy/docker/local_settings.py` is configured as http://mediacms.io, nginx-proxy has port 80 exposed. Clients can access the service on http://mediacms.io (Assuming DNS or the hosts file is setup correctly to point to the IP of the nginx-proxy instance). This is similar to [this deployment](../docker-compose-http-proxy.yaml).
+
+## Advanced Deployment, with reverse proxy, accessed as https://localhost
+
+The reverse proxy (`jwilder/nginx-proxy`) can be configured to provide SSL termination using self-signed certificates, letsencrypt or CA signed certificates (see: https://hub.docker.com/r/jwilder/nginx-proxy or [LetsEncrypt Example](https://www.singularaspect.com/use-nginx-proxy-and-letsencrypt-companion-to-host-multiple-websites/) ). In this case the FRONTEND_HOST should be set to https://mediacms.io. This is similar to [this deployment](../docker-compose-http-proxy.yaml).
+
+## A Scaleable Deployment Architecture (Docker, Swarm, Kubernetes)
+
+The architecture below generalises all the deployment scenarios above, and provides a conceptual design for other deployments based on kubernetes and docker swarm. It allows for horizontal scaleability through the use of multiple mediacms_web instances and celery_workers. For large deployments, managed postgres, redis and storage may be adopted.
+
+![MediaCMS](images/architecture.png)

install.sh

@@ -58,7 +58,7 @@ SECRET_KEY=`python -c 'from django.core.management.utils import get_random_secre
 FRONTEND_HOST=`echo "$FRONTEND_HOST" | sed -r 's/http:\/\///g'`
 FRONTEND_HOST=`echo "$FRONTEND_HOST" | sed -r 's/https:\/\///g'`
 
-sed -i s/localhost/$FRONTEND_HOST/g deploy/mediacms.io
+sed -i s/localhost/$FRONTEND_HOST/g deploy/local_install/mediacms.io
 
 
 echo 'FRONTEND_HOST='\'"$FRONTEND_HOST"\' >> cms/local_settings.py
@@ -66,6 +66,7 @@ echo 'PORTAL_NAME='\'"$PORTAL_NAME"\' >> cms/local_settings.py
 echo "SSL_FRONTEND_HOST = FRONTEND_HOST.replace('http', 'https')" >> cms/local_settings.py
 
 echo 'SECRET_KEY='\'"$SECRET_KEY"\' >> cms/local_settings.py
+echo "LOCAL_INSTALL = True" >> cms/local_settings.py
 
 mkdir logs
 mkdir pids
@@ -80,19 +81,19 @@ echo "from users.models import User; User.objects.create_superuser('admin', 'adm
 echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell
 
 chown -R www-data. /home/mediacms.io/
-cp deploy/celery_long.service /etc/systemd/system/celery_long.service && systemctl enable celery_long && systemctl start celery_long
-cp deploy/celery_short.service /etc/systemd/system/celery_short.service && systemctl enable celery_short && systemctl start celery_short
-cp deploy/celery_beat.service /etc/systemd/system/celery_beat.service && systemctl enable celery_beat &&systemctl start celery_beat
-cp deploy/mediacms.service /etc/systemd/system/mediacms.service && systemctl enable mediacms.service && systemctl start mediacms.service
+cp deploy/local_install/celery_long.service /etc/systemd/system/celery_long.service && systemctl enable celery_long && systemctl start celery_long
+cp deploy/local_install/celery_short.service /etc/systemd/system/celery_short.service && systemctl enable celery_short && systemctl start celery_short
+cp deploy/local_install/celery_beat.service /etc/systemd/system/celery_beat.service && systemctl enable celery_beat &&systemctl start celery_beat
+cp deploy/local_install/mediacms.service /etc/systemd/system/mediacms.service && systemctl enable mediacms.service && systemctl start mediacms.service
 
 mkdir -p /etc/letsencrypt/live/mediacms.io/
 mkdir -p /etc/letsencrypt/live/$FRONTEND_HOST
-cp deploy/mediacms.io_fullchain.pem /etc/letsencrypt/live/$FRONTEND_HOST/fullchain.pem
-cp deploy/mediacms.io_privkey.pem /etc/letsencrypt/live/$FRONTEND_HOST/privkey.pem
-cp deploy/mediacms.io /etc/nginx/sites-available/default
-cp deploy/mediacms.io /etc/nginx/sites-enabled/default
-cp deploy/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
-cp deploy/nginx.conf /etc/nginx/
+cp deploy/local_install/mediacms.io_fullchain.pem /etc/letsencrypt/live/$FRONTEND_HOST/fullchain.pem
+cp deploy/local_install/mediacms.io_privkey.pem /etc/letsencrypt/live/$FRONTEND_HOST/privkey.pem
+cp deploy/local_install/mediacms.io /etc/nginx/sites-available/default
+cp deploy/local_install/mediacms.io /etc/nginx/sites-enabled/default
+cp deploy/local_install/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
+cp deploy/local_install/nginx.conf /etc/nginx/
 systemctl stop nginx
 systemctl start nginx