diff --git a/Dockerfile b/Dockerfile index aa18557..8e36aa9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-buster AS compile-image +FROM python:3.11.4-bookworm AS compile-image SHELL ["/bin/bash", "-c"] @@ -24,7 +24,7 @@ RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-637.x86_64-u rm Bento4-SDK-1-6-0-637.x86_64-unknown-linux.zip ############ RUNTIME IMAGE ############ -FROM python:3.8-slim-buster as runtime-image +FROM python:3.11.4-bookworm as runtime-image ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/Dockerfile-dev b/Dockerfile-dev index 2547fc1..af64610 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,4 +1,4 @@ -FROM mediacms/mediacms:latest +FROM python:3.11.4-bookworm AS compile-image SHELL ["/bin/bash", "-c"] @@ -7,10 +7,65 @@ ENV VIRTUAL_ENV=/home/mediacms.io ENV PATH="$VIRTUAL_ENV/bin:$PATH" ENV PIP_NO_CACHE_DIR=1 -RUN cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV +RUN mkdir -p /home/mediacms.io/mediacms/{logs} && cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV +# Install dependencies: COPY requirements.txt . COPY requirements-dev.txt . RUN pip install -r requirements-dev.txt + +COPY . /home/mediacms.io/mediacms WORKDIR /home/mediacms.io/mediacms + +RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-637.x86_64-unknown-linux.zip && \ + unzip Bento4-SDK-1-6-0-637.x86_64-unknown-linux.zip -d ../bento4 && \ + mv ../bento4/Bento4-SDK-1-6-0-637.x86_64-unknown-linux/* ../bento4/ && \ + rm -rf ../bento4/Bento4-SDK-1-6-0-637.x86_64-unknown-linux && \ + rm -rf ../bento4/docs && \ + rm Bento4-SDK-1-6-0-637.x86_64-unknown-linux.zip + +############ RUNTIME IMAGE ############ +FROM python:3.11.4-bookworm as runtime-image + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +# See: https://github.com/celery/celery/issues/6285#issuecomment-715316219 +ENV CELERY_APP='cms' + +# Use these to toggle which processes supervisord should run +ENV ENABLE_UWSGI='yes' +ENV ENABLE_NGINX='yes' +ENV ENABLE_CELERY_BEAT='yes' +ENV ENABLE_CELERY_SHORT='yes' +ENV ENABLE_CELERY_LONG='yes' +ENV ENABLE_MIGRATIONS='yes' + +# Set up virtualenv +ENV VIRTUAL_ENV=/home/mediacms.io +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY --chown=www-data:www-data --from=compile-image /home/mediacms.io /home/mediacms.io + +RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends \ + supervisor nginx imagemagick procps wget xz-utils -y && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get purge --auto-remove && \ + apt-get clean + +RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \ + mkdir -p ffmpeg-tmp && \ + tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C ffmpeg-tmp && \ + cp -v ffmpeg-tmp/ffmpeg ffmpeg-tmp/ffprobe ffmpeg-tmp/qt-faststart /usr/local/bin && \ + rm -rf ffmpeg-tmp ffmpeg-release-amd64-static.tar.xz + +WORKDIR /home/mediacms.io/mediacms + +EXPOSE 9000 80 + +RUN chmod +x ./deploy/docker/entrypoint.sh + +ENTRYPOINT ["./deploy/docker/entrypoint.sh"] + +CMD ["./deploy/docker/start.sh"] diff --git a/HISTORY.md b/HISTORY.md index 7974071..427d3a8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,15 @@ # History +## 3.0.0 + +### Features +- Updates Python/Django requirements and Dockerfile to use latest 3.11 Python - https://github.com/mediacms-io/mediacms/pull/826/files. This update requires some manual steps, for existing (not new) installations. Check the update section under the [Admin docs](https://github.com/mediacms-io/mediacms/blob/main/docs/admins_docs.md#2-server-installation), either for single server or for Docker Compose installations +- Upgrade postgres on Docker Compose - https://github.com/mediacms-io/mediacms/pull/749 + +### Fixes +- video player options for HLS - https://github.com/mediacms-io/mediacms/pull/832 +- AVI videos not correctly recognised as videos - https://github.com/mediacms-io/mediacms/pull/833 + ## 2.1.0 ### Fixes diff --git a/cms/settings.py b/cms/settings.py index 8fe4e01..7945215 100644 --- a/cms/settings.py +++ b/cms/settings.py @@ -485,3 +485,5 @@ if GLOBAL_LOGIN_REQUIRED: r'/accounts/confirm-email/.*/$', r'/api/v[0-9]+/', ] + +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' diff --git a/cms/urls.py b/cms/urls.py index 4470006..bdf038b 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -1,7 +1,7 @@ import debug_toolbar -from django.conf.urls import include, re_path +from django.conf.urls import include from django.contrib import admin -from django.urls import path +from django.urls import path, re_path from django.views.generic.base import TemplateView from drf_yasg import openapi from drf_yasg.views import get_schema_view diff --git a/deploy/local_install/celery_beat.service b/deploy/local_install/celery_beat.service index 63a1333..be1065b 100644 --- a/deploy/local_install/celery_beat.service +++ b/deploy/local_install/celery_beat.service @@ -8,15 +8,13 @@ User=www-data Group=www-data Restart=always RestartSec=10 -Environment=APP_DIR="/home/mediacms.io/mediacms" +WorkingDirectory=/home/mediacms.io/mediacms Environment=CELERY_BIN="/home/mediacms.io/bin/celery" -Environment=CELERY_APP="cms" Environment=CELERYD_PID_FILE="/home/mediacms.io/mediacms/pids/beat%n.pid" Environment=CELERYD_LOG_FILE="/home/mediacms.io/mediacms/logs/beat%N.log" Environment=CELERYD_LOG_LEVEL="INFO" -Environment=APP_DIR="/home/mediacms.io/mediacms" -ExecStart=/bin/sh -c '${CELERY_BIN} beat -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} --workdir=${APP_DIR}' +ExecStart=/bin/sh -c '${CELERY_BIN} -A cms beat --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL}' ExecStop=/bin/kill -s TERM $MAINPID [Install] diff --git a/deploy/local_install/celery_long.service b/deploy/local_install/celery_long.service index ab8ab8a..c5c9cf4 100644 --- a/deploy/local_install/celery_long.service +++ b/deploy/local_install/celery_long.service @@ -8,23 +8,21 @@ User=www-data Group=www-data Restart=always RestartSec=10 -Environment=APP_DIR="/home/mediacms.io/mediacms" +WorkingDirectory=/home/mediacms.io/mediacms Environment=CELERYD_NODES="long1" Environment=CELERY_QUEUE="long_tasks" Environment=CELERY_BIN="/home/mediacms.io/bin/celery" -Environment=CELERY_APP="cms" Environment=CELERYD_MULTI="multi" Environment=CELERYD_OPTS="-Ofair --prefetch-multiplier=1" Environment=CELERYD_PID_FILE="/home/mediacms.io/mediacms/pids/%n.pid" Environment=CELERYD_LOG_FILE="/home/mediacms.io/mediacms/logs/%N.log" Environment=CELERYD_LOG_LEVEL="INFO" -Environment=APP_DIR="/home/mediacms.io/mediacms" -ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} --workdir=${APP_DIR} -Q ${CELERY_QUEUE}' +ExecStart=/bin/sh -c '${CELERY_BIN} -A cms multi start ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} -Q ${CELERY_QUEUE}' -ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE}' +ExecStop=/bin/sh -c '${CELERY_BIN} -A cms multi stopwait ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE}' -ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} --workdir=${APP_DIR} -Q ${CELERY_QUEUE}' +ExecReload=/bin/sh -c '${CELERY_BIN} -A cms multi restart ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} -Q ${CELERY_QUEUE}' [Install] WantedBy=multi-user.target diff --git a/deploy/local_install/celery_short.service b/deploy/local_install/celery_short.service index 0ac108b..c46d009 100644 --- a/deploy/local_install/celery_short.service +++ b/deploy/local_install/celery_short.service @@ -8,14 +8,13 @@ User=www-data Group=www-data Restart=always RestartSec=10 -Environment=APP_DIR="/home/mediacms.io/mediacms" +WorkingDirectory=/home/mediacms.io/mediacms Environment=CELERYD_NODES="short1 short2" Environment=CELERY_QUEUE="short_tasks" # Absolute or relative path to the 'celery' command: Environment=CELERY_BIN="/home/mediacms.io/bin/celery" # App instance to use # comment out this line if you don't use an app -Environment=CELERY_APP="cms" # or fully qualified: #CELERY_APP="proj.tasks:app" # How to call manage.py @@ -28,13 +27,12 @@ Environment=CELERYD_OPTS="--soft-time-limit=300 -c10" Environment=CELERYD_PID_FILE="/home/mediacms.io/mediacms/pids/%n.pid" Environment=CELERYD_LOG_FILE="/home/mediacms.io/mediacms/logs/%N.log" Environment=CELERYD_LOG_LEVEL="INFO" -Environment=APP_DIR="/home/mediacms.io/mediacms" -ExecStart=/bin/sh -c '${CELERY_BIN} multi start ${CELERYD_NODES} -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} --workdir=${APP_DIR} -Q ${CELERY_QUEUE}' +ExecStart=/bin/sh -c '${CELERY_BIN} -A cms multi start ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} -Q ${CELERY_QUEUE}' -ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE}' +ExecStop=/bin/sh -c '${CELERY_BIN} -A cms multi stopwait ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE}' -ExecReload=/bin/sh -c '${CELERY_BIN} multi restart ${CELERYD_NODES} -A ${CELERY_APP} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} --workdir=${APP_DIR} -Q ${CELERY_QUEUE}' +ExecReload=/bin/sh -c '${CELERY_BIN} -A cms multi restart ${CELERYD_NODES} --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} --loglevel=${CELERYD_LOG_LEVEL} ${CELERYD_OPTS} -Q ${CELERY_QUEUE}' [Install] WantedBy=multi-user.target diff --git a/docs/admins_docs.md b/docs/admins_docs.md index e89d0e8..c8f5429 100644 --- a/docs/admins_docs.md +++ b/docs/admins_docs.md @@ -25,12 +25,12 @@ This page is created for MediaCMS administrators that are responsible for settin ## 2. Server Installation -The core dependencies are Python3, Django3, Celery, PostgreSQL, Redis, ffmpeg. Any system that can have these dependencies installed, can run MediaCMS. But we strongly suggest installing on Linux Ubuntu 18 or 20 versions. +The core dependencies are Python3, Django3, Celery, PostgreSQL, Redis, ffmpeg. Any system that can have these dependencies installed, can run MediaCMS. But we strongly suggest installing on Linux Ubuntu (tested on versions 20, 22). -Installation on a Ubuntu 18 or 20 system with git utility installed should be completed in a few minutes with the following steps. +Installation on an Ubuntu system with git utility installed should be completed in a few minutes with the following steps. Make sure you run it as user root, on a clear system, since the automatic script will install and configure the following services: Celery/PostgreSQL/Redis/Nginx and will override any existing settings. -Automated script - tested on Ubuntu 18, Ubuntu 20, and Debian Buster +Automated script - tested on Ubuntu 20, Ubuntu 22 and Debian Buster ```bash mkdir /home/mediacms.io && cd /home/mediacms.io/ @@ -49,10 +49,16 @@ If you've used the above way to install MediaCMS, update with the following: cd /home/mediacms.io/mediacms # enter mediacms directory source /home/mediacms.io/bin/activate # use virtualenv git pull # update code +pip install -r requirements.txt -U # run pip install to update python manage.py migrate # run Django migrations sudo systemctl restart mediacms celery_long celery_short # restart services ``` +### Update from version 2 to version 3 +Version 3 is using Django 4 and Celery 5, and needs a recent Python 3.x version. If you are updating from an older version, make sure Python is updated first. Version 2 could run on Python 3.6, but version 3 needs Python3.8 and higher. + + + ### Configuration Checkout the configuration section here. @@ -66,7 +72,7 @@ Database can be backed up with pg_dump and media_files on /home/mediacms.io/medi ## Installation Install a recent version of [Docker](https://docs.docker.com/get-docker/), and [Docker Compose](https://docs.docker.com/compose/install/). -For Ubuntu 18/20 systems this is: +For Ubuntu 20/22 systems this is: ```bash curl -fsSL https://get.docker.com -o get-docker.sh @@ -112,6 +118,18 @@ docker-compose down docker-compose up ``` +### Update from version 2 to version 3 +Version 3 is using Python 3.11 and PostgreSQL 15. If you are updating from an older version, that was using PostgreSQL 13, the automatic update will not work, as you will receive the following message when the PostgreSQL container starts: + +``` +db_1 | 2023-06-27 11:07:42.959 UTC [1] FATAL: database files are incompatible with server +db_1 | 2023-06-27 11:07:42.959 UTC [1] DETAIL: The data directory was initialized by PostgreSQL version 13, which is not compatible with this version 15.2. +``` + +At this point there are two options: either edit the Docker Compose file and make use of the existing postgres:13 image, or otherwise you have to perform the migration from postgresql 13 to version 15. More notes on https://github.com/mediacms-io/mediacms/pull/749 + + + ## Configuration Checkout the configuration docs here. diff --git a/files/tasks.py b/files/tasks.py index 005fc80..aee5b67 100644 --- a/files/tasks.py +++ b/files/tasks.py @@ -7,10 +7,11 @@ import tempfile from datetime import datetime, timedelta from celery import Task -from celery.decorators import task +from celery import shared_task as task from celery.exceptions import SoftTimeLimitExceeded from celery.signals import task_revoked -from celery.task.control import revoke + +# from celery.task.control import revoke from celery.utils.log import get_task_logger from django.conf import settings from django.core.cache import cache @@ -460,10 +461,11 @@ def check_running_states(): if (now - encoding.update_date).seconds > settings.RUNNING_STATE_STALE: media = encoding.media profile = encoding.profile - task_id = encoding.task_id + # task_id = encoding.task_id # terminate task - if task_id: - revoke(task_id, terminate=True) + # TODO: not imported + # if task_id: + # revoke(task_id, terminate=True) encoding.delete() media.encode(profiles=[profile]) # TODO: allign with new code + chunksize... diff --git a/files/urls.py b/files/urls.py index d40f2ac..943b3b7 100644 --- a/files/urls.py +++ b/files/urls.py @@ -1,7 +1,7 @@ from django.conf import settings -from django.conf.urls import include, re_path +from django.conf.urls import include from django.conf.urls.static import static -from django.urls import path +from django.urls import path, re_path from . import management_views, views from .feeds import IndexRSSFeed, SearchRSSFeed diff --git a/files/views.py b/files/views.py index ae08c41..ce90196 100644 --- a/files/views.py +++ b/files/views.py @@ -1,6 +1,5 @@ from datetime import datetime, timedelta -from celery.task.control import revoke from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -1396,5 +1395,6 @@ class TaskDetail(APIView): permission_classes = (permissions.IsAdminUser,) def delete(self, request, uid, format=None): - revoke(uid, terminate=True) + # This is not imported! + # revoke(uid, terminate=True) return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/install.sh b/install.sh index 7143073..9aedaba 100644 --- a/install.sh +++ b/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# should be run as root and only on Ubuntu 18/20, Debian 10/11 (Buster/Bullseye) versions! +# should be run as root and only on Ubuntu 20/22, Debian 10/11 (Buster/Bullseye) versions! echo "Welcome to the MediacMS installation!"; if [ `id -u` -ne 0 ] @@ -22,11 +22,11 @@ done osVersion=$(lsb_release -d) -if [[ $osVersion == *"Ubuntu 20"* ]] || [[ $osVersion == *"Ubuntu 18"* ]] || [[ $osVersion == *"buster"* ]] || [[ $osVersion == *"bullseye"* ]]; then +if [[ $osVersion == *"Ubuntu 20"* ]] || [[ $osVersion == *"Ubuntu 22"* ]] || [[ $osVersion == *"buster"* ]] || [[ $osVersion == *"bullseye"* ]]; then echo 'Performing system update and dependency installation, this will take a few minutes' apt-get update && apt-get -y upgrade && apt-get install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip imagemagick python3-certbot-nginx certbot wget xz-utils -y else - echo "This script is tested for Ubuntu 18 and 20 versions only, if you want to try MediaCMS on another system you have to perform the manual installation" + echo "This script is tested for Ubuntu 20/22 versions only, if you want to try MediaCMS on another system you have to perform the manual installation" exit fi diff --git a/requirements.txt b/requirements.txt index be9c4e8..acbb677 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,21 @@ -Django==3.1.12 -djangorestframework==3.12.2 -django-allauth==0.44.0 -psycopg2-binary==2.8.6 -uwsgi==2.0.19.1 -django-redis==4.12.1 -celery==4.4.7 -drf-yasg==1.20.0 -Pillow==8.2.0 +Django==4.2.2 +djangorestframework==3.14.0 +django-allauth==0.54.0 +psycopg==3.1.9 +uwsgi==2.0.21 +django-redis==5.3.0 +celery==5.3.1 +drf-yasg==1.21.6 +Pillow==9.5.0 django-imagekit==4.1.0 -markdown==3.3.6 -django-filter==21.1 -filetype==1.0.10 -django-mptt==0.13.4 +markdown==3.4.3 +django-filter==23.2 +filetype==1.2.0 +django-mptt==0.14.0 django-crispy-forms==1.13.0 -requests==2.25.0 +requests==2.31.0 django-celery-email==3.0.0 -m3u8==1.0.0 -django-ckeditor==6.2.0 -django-debug-toolbar==3.2.4 -django-login-required-middleware==0.6.1 +m3u8==3.5.0 +django-ckeditor==6.6.1 +django-debug-toolbar==4.1.0 +django-login-required-middleware==0.9.0 diff --git a/uploader/urls.py b/uploader/urls.py index bee9761..71265cf 100644 --- a/uploader/urls.py +++ b/uploader/urls.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from django.conf.urls import re_path +from django.urls import re_path from . import views diff --git a/users/urls.py b/users/urls.py index 4676c49..c664878 100644 --- a/users/urls.py +++ b/users/urls.py @@ -1,5 +1,4 @@ -from django.conf.urls import re_path -from django.urls import path +from django.urls import path, re_path from . import views diff --git a/version.py b/version.py index 2101409..ea9d694 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -VERSION = "2.0.0" +VERSION = "3.0.0"