initial docker deployment implementation

.gitignore

@@ -0,0 +1,3 @@
+media_files/encoded/
+media_files/original/
+postgres_data/

.mailmap

@@ -0,0 +1 @@
+Swift Ugandan <swiftugandan@gmail.com> <swiftugandan@gmail.com>

Dockerfile

@@ -0,0 +1,66 @@
+FROM python:3.8-buster AS compile-image
+
+SHELL ["/bin/bash", "-c"]
+
+# Set up virtualenv
+ENV VIRTUAL_ENV=/home/mediacms.io
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ENV PIP_NO_CACHE_DIR=1
+
+RUN mkdir -p /home/mediacms.io/mediacms/{logs,pids} && cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV
+
+# Install dependencies:
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . /home/mediacms.io/mediacms
+WORKDIR /home/mediacms.io/mediacms
+
+RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip && \
+    mkdir -p /home/mediacms.io/mediacms/media_files/hls Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/ && \
+    unzip -j Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/mp4hls -d Bento4-SDK-1-6-0-632.x86_64-unknown-linux/bin/ && \
+    rm Bento4-SDK-1-6-0-632.x86_64-unknown-linux.zip
+
+RUN chown -R www-data. /home/mediacms.io/ && chmod +x /home/mediacms.io/mediacms/deploy/docker/start.sh /home/mediacms.io/mediacms/deploy/docker/prestart.sh
+
+############ RUNTIME IMAGE ############
+FROM python:3.8-slim-buster as runtime-image
+
+ENV PYTHONUNBUFFERED=1
+ENV ADMIN_USER='admin'
+ENV ADMIN_PASSWORD='mediacms'
+ENV ADMIN_EMAIL='admin@localhost'
+
+# See: https://github.com/celery/celery/issues/6285#issuecomment-715316219
+ENV CELERY_APP='cms'
+
+# Use these to toggle which processes supervisord should run
+ENV ENABLE_UWSGI='yes'
+ENV ENABLE_NGINX='yes'
+ENV ENABLE_CELERY_BEAT='yes'
+ENV ENABLE_CELERY_SHORT='yes'
+ENV ENABLE_CELERY_LONG='yes'
+ENV ENABLE_MIGRATIONS='yes'
+
+# Set up virtualenv
+ENV VIRTUAL_ENV=/home/mediacms.io
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+COPY --from=compile-image /home/mediacms.io /home/mediacms.io
+
+RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends \
+    supervisor nginx ffmpeg imagemagick procps -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get purge --auto-remove && \
+    apt-get clean
+
+# forward request and error logs to docker log collector
+RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log && \
+    ln -sf /dev/stdout /var/log/nginx/mediacms.io.access.log && ln -sf /dev/stderr /var/log/nginx/mediacms.io.error.log
+
+WORKDIR /home/mediacms.io/mediacms
+
+EXPOSE 9000 80
+
+CMD ["./deploy/docker/start.sh"]
+  

cms/settings.py

@@ -419,6 +419,8 @@ CELERY_BEAT_SCHEDULE = {
 # TODO: beat, delete chunks from media root
 # chunks_dir after xx days...(also uploads_dir)
 
+LOCAL_INSTALL = False
+
 try:
     # keep a local_settings.py file for local overrides
     from .local_settings import *
@@ -434,4 +436,7 @@ if "http" not in FRONTEND_HOST:
     # FRONTEND_HOST needs a http:// preffix
     FRONTEND_HOST = f"http://{FRONTEND_HOST}"
 
-SSL_FRONTEND_HOST = FRONTEND_HOST.replace("http", "https")
+if LOCAL_INSTALL:
+    SSL_FRONTEND_HOST = FRONTEND_HOST.replace("http", "https")
+else:
+    SSL_FRONTEND_HOST = FRONTEND_HOST

deploy/docker/README.md

@@ -0,0 +1,3 @@
+# MediaCMS on Docker
+
+See: [Details](../../docs/Docker_deployment.md)

deploy/docker/local_settings.py

@@ -0,0 +1,32 @@
+FRONTEND_HOST = 'http://localhost'
+PORTAL_NAME = 'MediaCMS'
+SECRET_KEY = 'ma!s3^b-cw!f#7s6s0m3*jx77a@riw(7701**(r=ww%w!2+yk2'
+POSTGRES_HOST = 'db'
+REDIS_LOCATION = "redis://redis:6379/1"
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": "mediacms",
+        "HOST": POSTGRES_HOST,
+        "PORT": "5432",
+        "USER": "mediacms",
+        "PASSWORD": "mediacms",
+    }
+}
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_LOCATION,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    }
+}
+
+# CELERY STUFF
+BROKER_URL = REDIS_LOCATION
+CELERY_RESULT_BACKEND = BROKER_URL
+
+DEBUG = False

deploy/docker/nginx_http_only.conf

@@ -0,0 +1,30 @@
+server {
+    listen 80 ;
+
+    gzip on;
+    access_log /var/log/nginx/mediacms.io.access.log;
+
+    error_log  /var/log/nginx/mediacms.io.error.log  warn;
+
+    location /static {
+        alias /home/mediacms.io/mediacms/static ;
+    }
+
+    location /media/original {
+        alias /home/mediacms.io/mediacms/media_files/original;
+    }
+
+    location /media {
+        alias /home/mediacms.io/mediacms/media_files ;
+    }
+
+    location / {
+        add_header 'Access-Control-Allow-Origin' '*';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+        add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
+        add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
+
+        include /etc/nginx/sites-enabled/uwsgi_params;
+        uwsgi_pass 127.0.0.1:9000;
+    }
+}

deploy/docker/prestart.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+mkdir -p /home/mediacms.io/mediacms/logs
+touch /home/mediacms.io/mediacms/logs/debug.log
+chown www-data. -R /home/mediacms.io/mediacms/logs
+
+RANDOM_ADMIN_PASS=`python -c "import secrets;chars = 'abcdefghijklmnopqrstuvwxyz0123456789';print(''.join(secrets.choice(chars) for i in range(10)))"`
+ADMIN_PASSWORD=${ADMIN_PASSWORD:-$RANDOM_ADMIN_PASS}
+
+if [ X"$ENABLE_MIGRATIONS" = X"yes" ]; then
+    python manage.py migrate
+    python manage.py loaddata fixtures/encoding_profiles.json
+    python manage.py loaddata fixtures/categories.json
+    python manage.py collectstatic --noinput
+
+    echo "Admin Password: $ADMIN_PASSWORD"
+
+    # post_save, needs redis to succeed (ie. migrate depends on redis)
+    DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
+        --no-input \
+        --username=$ADMIN_USER \
+        --email=$ADMIN_EMAIL \
+        --database=default || true
+
+    # echo "Updating hostname ..."
+    # TODO: Get the FRONTEND_HOST from cms/local_settings.py
+    # echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell
+fi
+
+# Setting up internal nginx server
+# HTTPS setup is delegated to a reverse proxy running infront of the application
+
+cp deploy/docker/nginx_http_only.conf /etc/nginx/sites-available/default
+cp deploy/docker/nginx_http_only.conf /etc/nginx/sites-enabled/default
+cp deploy/docker/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
+cp deploy/docker/nginx.conf /etc/nginx/
+
+#### Supervisord Configurations #####
+
+cp deploy/docker/supervisord/supervisord-debian.conf /etc/supervisor/conf.d/supervisord-debian.conf
+
+if [ X"$ENABLE_UWSGI" = X"yes" ] ; then
+    echo "Enabling uwsgi app server"
+    cp deploy/docker/supervisord/supervisord-uwsgi.conf /etc/supervisor/conf.d/supervisord-uwsgi.conf
+fi
+
+if [ X"$ENABLE_NGINX" = X"yes" ] ; then
+    echo "Enabling nginx as uwsgi app proxy and media server"
+    cp deploy/docker/supervisord/supervisord-nginx.conf /etc/supervisor/conf.d/supervisord-nginx.conf
+fi
+
+if [ X"$ENABLE_CELERY_BEAT" = X"yes" ] ; then
+    echo "Enabling celery-beat scheduling server"
+    cp deploy/docker/supervisord/supervisord-celery_beat.conf /etc/supervisor/conf.d/supervisord-celery_beat.conf
+fi
+
+if [ X"$ENABLE_CELERY_SHORT" = X"yes" ] ; then
+    echo "Enabling celery-short task worker"
+    cp deploy/docker/supervisord/supervisord-celery_short.conf /etc/supervisor/conf.d/supervisord-celery_short.conf
+fi
+
+if [ X"$ENABLE_CELERY_LONG" = X"yes" ] ; then
+    echo "Enabling celery-long task worker"
+    cp deploy/docker/supervisord/supervisord-celery_long.conf /etc/supervisor/conf.d/supervisord-celery_long.conf
+fi

deploy/docker/reverse_proxy/certs/mediacms.io.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICyTCCAbGgAwIBAgIJAPHG6VrZeH1/MA0GCSqGSIb3DQEBBQUAMBYxFDASBgNV
+BAMTC21lZGlhY21zLmlvMB4XDTIxMDExNjE1NDUzNVoXDTMxMDExNDE1NDUzNVow
+FjEUMBIGA1UEAxMLbWVkaWFjbXMuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQC+xxnUwjoIZq8sTw2DYGgIYxQ5lJ8Uvt+z+K/PQpT5nFqd1jURF8Zd
+a92TlJjM5aSKosavuPHbFNkA7rSnLvP+I+8qsNPoinEUlE12Spg4E7dQkOkvTGty
+/amFq69o9vm46GpvwImTZ5AQkzejk0ARUuFSdq9ev4aA44IBYiV4c2jRqnP7LY4j
++SA/rt+9bNUTwQ6QWEHDTHmKePr91UTZBcDw/oaoaJwWFXuEVC7VjtDN09ZNjkdg
+pI6PvQZVw2IlBHS4S+ol+G2k2ckSCLgOj+dZrndr8OGrlAb8wgsInLK54nHm6VRe
+G883CJd/VlOQAulE26ZkzIdAIjJCwb+DAgMBAAGjGjAYMBYGA1UdEQQPMA2CC21l
+ZGlhY21zLmlvMA0GCSqGSIb3DQEBBQUAA4IBAQBwxkTE5GBuFjcFsBzMqhePgC7W
+INzoTmyMLJrNClFLkUKkDrwNmShLNhZUbMHeDD1W40aKYJCV44QhT04fK18HU/DW
+RkprlJDI8WUnuY97zN6Ms9z/GwYDGNXGLh8I/SEMhfJ8cIQuofhvuyi/E4AdWRva
+Hw1RSC8RikTZQ5Y84oJ44RfHNfK7xkaeurcm/Tn4Vxx4RgXA2MMoFA7XbT08vhKw
+iiQ9u4QL1GP3Nm8cTDDA9OChhLl56k24MD3WJM2HFTFlE5S4hFRkEqzy4pI/BTU4
+S4fkXK88xDtB/kHlHgRQiNH+6ik8ZXXP1F56+vDLuR28nK3hRTpQwaRQ7dzC
+-----END CERTIFICATE-----

deploy/docker/reverse_proxy/certs/mediacms.io.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvscZ1MI6CGavLE8Ng2BoCGMUOZSfFL7fs/ivz0KU+ZxandY1
+ERfGXWvdk5SYzOWkiqLGr7jx2xTZAO60py7z/iPvKrDT6IpxFJRNdkqYOBO3UJDp
+L0xrcv2phauvaPb5uOhqb8CJk2eQEJM3o5NAEVLhUnavXr+GgOOCAWIleHNo0apz
++y2OI/kgP67fvWzVE8EOkFhBw0x5inj6/dVE2QXA8P6GqGicFhV7hFQu1Y7QzdPW
+TY5HYKSOj70GVcNiJQR0uEvqJfhtpNnJEgi4Do/nWa53a/Dhq5QG/MILCJyyueJx
+5ulUXhvPNwiXf1ZTkALpRNumZMyHQCIyQsG/gwIDAQABAoIBAQCMauVTWOX3+wRi
+G4l5skLAMZTYUNDKJzdmMtvMNFiMZI258Mk8XIBvkI4VKuFQppH2TJrrCbhSJUUX
+z5p+FywVWYOWq3I9jXBv0jw1ne/uDmz1ysMnQhswFw5oSZahLm9drwtwV4mrSiWa
+XZEtP5t/ZL5dwOeRWGz6fvnYZNHpldkyZDO8+ywB55P+XDfGyxUppMOWdbNV9wGo
+Fg6ypUFWFEUD8Ou8xd5FT5QqrQ5ruZJDKcYhPTuK0/dRsMgAxB+Bhf8XH3ynUZp2
++qMXcKyIQumq9r+/ulE/Yhnbh/E4hYBbThhnmPejNeSvWb7niYfL/fsPI8FLmtmi
+z+Ab5IABAoGBAPb4rUP3rVDatzsf2jJSUXcMn9gAdf3ajbw+Z2CCf4j1wj1BTIM5
+5YmABJMS7D97H+a6Vn+SZd426UJYMlKPDnVOTXvvlzhP/TmiSFa8FW8Rjho9Rcnc
+LDwnO48q0AJg3HslrjEUaDuWaNHJkqB5tGqzKgZCZxrqoNRYgufPwH+DAoGBAMXA
+hr4KxwlcXYIwbM+Uj8eHnESwPWk2+cRwpv62u4ezctZrBCHgAHKvznG7VUeniQfj
+P2MaGFz6Pvzw3cFRLKRVqJom5iXO6+H0EucusdqJY4xdWZt02ZweoJXZi9tiDGmG
+fPOp3vUax4uGUS4LeSo+ZhPNfbfy9c8ZGQ7Z9cABAoGAA0oyvKoK8/3F3RLCjFMO
+ZMCVTIJNEBGeO7i1FdMHMeLcMIazJzhZN2iuJutknD/en+sxhceEdd5TYx/bo7/m
+GGfvnkwFvqlKHT9tKUKeInmgY/cW++Zj7HU1VOXkGXQC290Xoe28qbaKNOkze9HD
+NnymfajayMABXnLDY6Uf0lMCgYBgVLIOn4dnuvPeOKK42ADWTOxF1aiEuYAgPlRL
+Hk7qAvN9GfKQYeM1+whRBNW9KxKoof290/dsS4clhlwwEM/zWbrhJPPWFR95GYGf
+1nJTJ7wzo0HEZb6fu5e0h54Gh5POT/JMbEKtGZd9Ezg2euZSOsVU/jQwyI0PjoVT
+Y7/AAQKBgHEWxqZwJ3BE9gXRRGhKiPLNG9+OzwjrMiinN4s9Hol0STB1AqryswpU
+9QL7Mb8SdxCDY2CseQRwT2VFiP10ElCuzZJ6Yk2cxzmrxAAhRWGHSIuNlnXTA6LR
+AhIMLFLz+7KqBx8VHybkhZNCR1nPR9MicS9MpSEYTuRAnV2B1cuU
+-----END RSA PRIVATE KEY-----

deploy/docker/reverse_proxy/client_max_body_size.conf

@@ -0,0 +1 @@
+client_max_body_size 1g;

deploy/docker/start.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env sh
+set -e
+
+# If there's a prestart.sh script in the /app directory, run it before starting
+PRE_START_PATH=deploy/docker/prestart.sh
+echo "Checking for script in $PRE_START_PATH"
+if [ -f $PRE_START_PATH ] ; then
+    echo "Running script $PRE_START_PATH"
+    . $PRE_START_PATH
+else
+    echo "There is no script $PRE_START_PATH"
+fi
+
+# Start Supervisor, with Nginx and uWSGI
+echo "Starting server using supervisord..."
+
+exec /usr/bin/supervisord

deploy/docker/supervisord/supervisord-celery_beat.conf

@@ -0,0 +1,12 @@
+[program:celery_beat]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=0
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=300
+startinorder=true
+command=/home/mediacms.io/bin/celery beat --pidfile=/home/mediacms.io/mediacms/pids/beat%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_beat.log

deploy/docker/supervisord/supervisord-celery_long.conf

@@ -0,0 +1,13 @@
+[program:celery_long]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=10
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=500
+startinorder=true
+startsecs=0
+command=/home/mediacms.io/bin/celery multi start long1 --pidfile=/home/mediacms.io/mediacms/pids/%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_long.log -Ofair --prefetch-multiplier=1 -Q long_tasks

deploy/docker/supervisord/supervisord-celery_short.conf

@@ -0,0 +1,12 @@
+[program:celery_short]
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+startsecs=0
+numprocs=1
+user=www-data
+directory=/home/mediacms.io/mediacms
+priority=400
+startinorder=true
+command=/home/mediacms.io/bin/celery multi start short1 short2 --pidfile=/home/mediacms.io/mediacms/pids/%%n.pid --loglevel=INFO --logfile=/home/mediacms.io/mediacms/logs/celery_short.log --soft-time-limit=300 -c10 -Q short_tasks

deploy/docker/supervisord/supervisord-debian.conf

@@ -0,0 +1,2 @@
+[supervisord]
+nodaemon=true

deploy/docker/supervisord/supervisord-nginx.conf

@@ -0,0 +1,11 @@
+[program:nginx]
+command=/usr/sbin/nginx -g 'daemon off;'
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=200
+startinorder=true
+startsecs=0
+# Graceful stop, see http://nginx.org/en/docs/control.html
+stopsignal=QUIT

deploy/docker/supervisord/supervisord-uwsgi.conf

@@ -0,0 +1,9 @@
+[program:uwsgi]
+command=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/deploy/docker/uwsgi.ini
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+priority=100
+startinorder=true
+startsecs=0

deploy/docker/uwsgi.ini

@@ -0,0 +1,24 @@
+[uwsgi]
+
+chdir = /home/mediacms.io/mediacms/
+virtualenv = /home/mediacms.io
+module = cms.wsgi
+
+uid=www-data
+gid=www-data
+
+processes = 2
+threads = 2
+
+master = true
+
+socket = 127.0.0.1:9000
+
+workers = 2
+
+vacuum = true
+
+hook-master-start = unix_signal:15 gracefully_kill_them_all
+need-app = true
+die-on-term = true
+

deploy/local_install/mediacms.io

@@ -59,7 +59,7 @@ server {
     location /media/original {
         alias /home/mediacms.io/mediacms/media_files/original;
         #auth_basic "auth protected area";
-        #auth_basic_user_file /home/mediacms.io/mediacms/deploy/.htpasswd;
+        #auth_basic_user_file /home/mediacms.io/mediacms/deploy/local_install/.htpasswd;
     }
 
     location /media {

deploy/local_install/mediacms.service

@@ -2,7 +2,7 @@
 Description=MediaCMS uwsgi
 
 [Service]
-ExecStart=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/uwsgi.ini
+ExecStart=/home/mediacms.io/bin/uwsgi --ini /home/mediacms.io/mediacms/deploy/local_install/uwsgi.ini
 ExecStop=/usr/bin/killall -9 uwsgi
 RestartSec=3
 #ExecRestart=killall -9 uwsgi; sleep 5; /home/sss/bin/uwsgi --ini /home/sss/wordgames/uwsgi.ini

deploy/local_install/nginx.conf

@@ -0,0 +1,41 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 10240;
+}
+
+    worker_rlimit_nofile    20000;  #each connection needs a filehandle (or 2 if you are proxying)
+http {
+    proxy_connect_timeout 75;
+    proxy_read_timeout 12000;
+    client_max_body_size 5800M;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 10;
+    types_hash_max_size 2048;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+    ssl_prefer_server_ciphers on;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    gzip on;
+    gzip_disable "msie6";
+
+    log_format compression '$remote_addr - $remote_user [$time_local] '
+                           '"$request" $status $body_bytes_sent '
+                           '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+    include /etc/nginx/conf.d/*.conf;
+    include /etc/nginx/sites-enabled/*;
+}
+

deploy/local_install/uwsgi.ini

@@ -1,8 +1,8 @@
 [uwsgi]
 
-chdir           = /home/mediacms.io/mediacms/
+chdir = /home/mediacms.io/mediacms/
 virtualenv = /home/mediacms.io
-module          = cms.wsgi
+module = cms.wsgi
 
 uid=www-data
 gid=www-data
@@ -10,7 +10,7 @@ gid=www-data
 processes = 2
 threads = 2
 
-master          = true
+master = true
 
 socket = 127.0.0.1:9000
 #socket = /home/mediacms.io/mediacms/deploy/uwsgi.sock
@@ -19,7 +19,7 @@ socket = 127.0.0.1:9000
 workers = 2
 
 
-vacuum          = true
+vacuum = true
 
 logto = /home/mediacms.io/mediacms/logs/errorlog.txt
 

deploy/local_install/uwsgi_params

@@ -0,0 +1,16 @@
+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;

deploy/uwsgi.ini

@@ -1,19 +0,0 @@
-[uwsgi]
-
-chdir = /home/mediacms.io/mediacms/
-virtualenv = /home/mediacms.io
-module = cms.wsgi
-
-uid = www-data
-gid = www-data
-
-processes = 10
-threads = 10
-master = true
-workers = 8
-vacuum = true
-
-socket = 127.0.0.1:9000
-
-logto = /home/mediacms.io/mediacms/logs/errorlog.txt
-

docker-compose-http-proxy.yaml

@@ -0,0 +1,93 @@
+version: "3"
+
+services:
+  nginx-proxy:
+    image: jwilder/nginx-proxy
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - ./deploy/docker/reverse_proxy/client_max_body_size.conf:/etc/nginx/conf.d/client_max_body_size.conf:ro
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./media_files/:/home/mediacms.io/mediacms/media_files/
+      - ./static/:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+      VIRTUAL_HOST: localhost
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./media_files/:/home/mediacms.io/mediacms/media_files/
+      - ./static/:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - ./postgres_data/:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

docker-compose-https-proxy.yaml

@@ -0,0 +1,95 @@
+version: "3"
+
+services:
+  nginx-proxy:
+    image: jwilder/nginx-proxy
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - ./deploy/docker/reverse_proxy/certs:/etc/nginx/certs
+      - ./deploy/docker/reverse_proxy/client_max_body_size.conf:/etc/nginx/conf.d/client_max_body_size.conf:ro
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./media_files/:/home/mediacms.io/mediacms/media_files/
+      - ./static/:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+      VIRTUAL_HOST: localhost
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 2
+    volumes:
+      - ./media_files/:/home/mediacms.io/mediacms/media_files/
+      - ./static/:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - ./postgres_data/:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

docker-compose-named-volumes.yaml

@@ -0,0 +1,91 @@
+version: "3"
+
+services:
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    ports:
+      - "80:80"
+    volumes:
+      - media_store:/home/mediacms.io/mediacms/media_files/
+      - static_store:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 2
+    volumes:
+      - media_store:/home/mediacms.io/mediacms/media_files/
+      - static_store:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - postgres_data:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+volumes:
+  postgres_data:
+  media_store:
+  static_store:

docker-compose.yaml

@@ -0,0 +1,87 @@
+version: "3"
+
+services:
+  migrations:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+    depends_on:
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
+  web:
+    build:
+      context: .
+      target: runtime-image
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    ports:
+      - "80:80"
+    volumes:
+      - ./media_files/:/home/mediacms.io/mediacms/media_files/
+      - ./static/:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  celery_beat:
+    image: mediacms:latest
+    volumes:
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_SHORT: 'no'
+      ENABLE_CELERY_LONG: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - redis
+  celery_worker:
+    image: mediacms:latest
+    deploy:
+      replicas: 1
+    volumes:
+      - ./media_files/:/home/mediacms.io/mediacms/media_files/
+      - ./static/:/home/mediacms.io/mediacms/static/
+      - ./deploy/docker/local_settings.py:/home/mediacms.io/mediacms/cms/local_settings.py
+    environment:
+      ENABLE_UWSGI: 'no'
+      ENABLE_NGINX: 'no'
+      ENABLE_CELERY_BEAT: 'no'
+      ENABLE_MIGRATIONS: 'no'
+    depends_on:
+      - migrations
+  db:
+    image: postgres
+    volumes:
+      - ./postgres_data:/var/lib/postgresql/data/
+    restart: always
+    environment:
+      POSTGRES_USER: mediacms
+      POSTGRES_PASSWORD: mediacms
+      POSTGRES_DB: mediacms
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mediacms"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  redis:
+    image: "redis:alpine"
+    restart: always
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

docs/Docker_deployment.md

@@ -0,0 +1,40 @@
+# MediaCMS on Docker
+
+The mediacms image is built to use supervisord as the main process, which manages one or more services required to run mediacms. We can toggle which services are run in a given container by setting the environment variables below to `yes` or `no`:
+
+* ENABLE_UWSGI
+* ENABLE_NGINX
+* ENABLE_CELERY_BEAT
+* ENABLE_CELERY_SHORT
+* ENABLE_CELERY_LONG
+* ENABLE_MIGRATIONS
+
+By default, all these services are enabled, but in order to create a scaleable deployment, some of them are disabled.
+
+Also see the `Dockerfile` for other environment variables which you may wish to override. Application settings can also be overridden by updating the `deploy/docker/local_settings.py` file.
+
+See example deployments in the sections below. These example deployments have been tested on `docker-compose version 1.27.4` running on `Docker version 19.03.13`
+
+To run, update the configs above if necessary, build the image by running `docker-compose build`, then run `docker-compose run`
+
+## Simple Deployment, accessed as http://localhost
+
+The main container runs migrations, mediacms_web, celery_beat, celery_workers (celery_short and celery_long services), exposed on port 80 supported by redis and postgres database. The FRONTEND_HOST in `deploy/docker/local_settings.py` is configured as http://localhost, on the docker host machine.
+
+## Advanced Deployment, accessed as http://localhost:8000
+
+Here we can run 1 mediacms_web instance, with the FRONTEND_HOST in `deploy/docker/local_settings.py` is configured as http://localhost:8000. This is bootstrapped by a single migrations instance and supported by a single celery_beat instance and 1 or more celery_worker instances. Redis and postgres containers are also used for persistence. Clients can access the service on http://localhost:8000, on the docker host machine. This is similar to [this deployment](../docker-compose.yaml), with a `port` defined in FRONTEND_HOST.
+
+## Advanced Deployment, with reverse proxy, accessed as http://mediacms.io
+
+Here we can use `jwilder/nginx-proxy` to reverse proxy to 1 or more instances of mediacms_web supported by other services as mentioned in the previous deployment. The FRONTEND_HOST in `deploy/docker/local_settings.py` is configured as http://mediacms.io, nginx-proxy has port 80 exposed. Clients can access the service on http://mediacms.io (Assuming DNS or the hosts file is setup correctly to point to the IP of the nginx-proxy instance). This is similar to [this deployment](../docker-compose-http-proxy.yaml).
+
+## Advanced Deployment, with reverse proxy, accessed as https://mediacms.io
+
+The reverse proxy (`jwilder/nginx-proxy`) can be configured to provide SSL termination using self-signed certificates, letsencrypt or CA signed certificates (see: https://hub.docker.com/r/jwilder/nginx-proxy). In this case the FRONTEND_HOST should be set to https://mediacms.io. This is similar to [this deployment](../docker-compose-http-proxy.yaml).
+
+## A Scaleable Deployment Architecture (Docker, Swarm, Kubernetes)
+
+The architecture below generalises all the deployment scenarios above, and provides a conceptual design for other deployments based on kubernetes and docker swarm. It allows for horizontal scaleability through the use of multiple mediacms_web instances and celery_workers. For large deployments, managed postgres, redis and storage may be adopted.
+
+![MediaCMS](images/architecture.png)

files/context_processors.py

@@ -5,7 +5,7 @@ from .methods import is_mediacms_editor, is_mediacms_manager
 def stuff(request):
     """Pass settings to the frontend"""
     ret = {}
-    if request.is_secure():
+    if request.is_secure() and settings.LOCAL_INSTALL:
         # in case session is https, pass this setting so
         # that the frontend uses https too
         ret["FRONTEND_HOST"] = settings.SSL_FRONTEND_HOST

install.sh

@@ -58,7 +58,7 @@ SECRET_KEY=`python -c 'from django.core.management.utils import get_random_secre
 FRONTEND_HOST=`echo "$FRONTEND_HOST" | sed -r 's/http:\/\///g'`
 FRONTEND_HOST=`echo "$FRONTEND_HOST" | sed -r 's/https:\/\///g'`
 
-sed -i s/localhost/$FRONTEND_HOST/g deploy/mediacms.io
+sed -i s/localhost/$FRONTEND_HOST/g deploy/local_install/mediacms.io
 
 
 echo 'FRONTEND_HOST='\'"$FRONTEND_HOST"\' >> cms/local_settings.py
@@ -66,6 +66,7 @@ echo 'PORTAL_NAME='\'"$PORTAL_NAME"\' >> cms/local_settings.py
 echo "SSL_FRONTEND_HOST = FRONTEND_HOST.replace('http', 'https')" >> cms/local_settings.py
 
 echo 'SECRET_KEY='\'"$SECRET_KEY"\' >> cms/local_settings.py
+echo "LOCAL_INSTALL = True" >> cms/local_settings.py
 
 mkdir logs
 mkdir pids
@@ -80,19 +81,19 @@ echo "from users.models import User; User.objects.create_superuser('admin', 'adm
 echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell
 
 chown -R www-data. /home/mediacms.io/
-cp deploy/celery_long.service /etc/systemd/system/celery_long.service && systemctl enable celery_long && systemctl start celery_long
-cp deploy/celery_short.service /etc/systemd/system/celery_short.service && systemctl enable celery_short && systemctl start celery_short
-cp deploy/celery_beat.service /etc/systemd/system/celery_beat.service && systemctl enable celery_beat &&systemctl start celery_beat
-cp deploy/mediacms.service /etc/systemd/system/mediacms.service && systemctl enable mediacms.service && systemctl start mediacms.service
+cp deploy/local_install/celery_long.service /etc/systemd/system/celery_long.service && systemctl enable celery_long && systemctl start celery_long
+cp deploy/local_install/celery_short.service /etc/systemd/system/celery_short.service && systemctl enable celery_short && systemctl start celery_short
+cp deploy/local_install/celery_beat.service /etc/systemd/system/celery_beat.service && systemctl enable celery_beat &&systemctl start celery_beat
+cp deploy/local_install/mediacms.service /etc/systemd/system/mediacms.service && systemctl enable mediacms.service && systemctl start mediacms.service
 
 mkdir -p /etc/letsencrypt/live/mediacms.io/
 mkdir -p /etc/letsencrypt/live/$FRONTEND_HOST
-cp deploy/mediacms.io_fullchain.pem /etc/letsencrypt/live/$FRONTEND_HOST/fullchain.pem
-cp deploy/mediacms.io_privkey.pem /etc/letsencrypt/live/$FRONTEND_HOST/privkey.pem
-cp deploy/mediacms.io /etc/nginx/sites-available/default
-cp deploy/mediacms.io /etc/nginx/sites-enabled/default
-cp deploy/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
-cp deploy/nginx.conf /etc/nginx/
+cp deploy/local_install/mediacms.io_fullchain.pem /etc/letsencrypt/live/$FRONTEND_HOST/fullchain.pem
+cp deploy/local_install/mediacms.io_privkey.pem /etc/letsencrypt/live/$FRONTEND_HOST/privkey.pem
+cp deploy/local_install/mediacms.io /etc/nginx/sites-available/default
+cp deploy/local_install/mediacms.io /etc/nginx/sites-enabled/default
+cp deploy/local_install/uwsgi_params /etc/nginx/sites-enabled/uwsgi_params
+cp deploy/local_install/nginx.conf /etc/nginx/
 systemctl stop nginx
 systemctl start nginx