From d63e7677e4c1bcd2cff1eefdd226e683e3d9adaa Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Mon, 17 Jul 2017 22:00:32 +0200 Subject: [PATCH] WIP: Debian installation instructions --- .gitlab-ci.yml | 2 + api/compose/django/entrypoint.sh | 4 +- api/config/settings/common.py | 46 +-- api/config/settings/production.py | 13 +- api/config/settings/test.py | 3 + .../management/commands/import_files.py | 4 +- .../audiofile/{importer.py => tasks.py} | 0 .../audiofile/tests/test_disk_import.py | 4 +- api/manage.py | 5 +- api/requirements.apt | 43 +-- api/test.yml | 2 + deploy/env.prod.sample | 54 +++- deploy/funkwhale-server.service | 14 + deploy/funkwhale-worker.service | 14 + deploy/funkwhale.target | 3 + deploy/nginx.conf | 26 +- dev.yml | 6 +- docs/changelog.rst | 4 +- docs/conf.py | 17 +- docs/importing-music.rst | 33 ++- docs/installation/debian.rst | 280 ++++++++++++++++++ docs/installation/docker.rst | 8 +- docs/installation/external_dependencies.rst | 62 ++++ docs/installation/index.rst | 11 +- docs/installation/systemd.rst | 42 +++ 25 files changed, 574 insertions(+), 126 deletions(-) rename api/funkwhale_api/providers/audiofile/{importer.py => tasks.py} (100%) create mode 100644 deploy/funkwhale-server.service create mode 100644 deploy/funkwhale-worker.service create mode 100644 deploy/funkwhale.target create mode 100644 docs/installation/debian.rst create mode 100644 docs/installation/external_dependencies.rst create mode 100644 docs/installation/systemd.rst diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 24b23b84b..0b68d19c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,8 @@ test_api: - pip install -r requirements/test.txt script: - pytest + variables: + DATABASE_URL: "sqlite://" tags: - docker diff --git a/api/compose/django/entrypoint.sh b/api/compose/django/entrypoint.sh index 98b3681e1..7e789968b 100755 --- a/api/compose/django/entrypoint.sh +++ b/api/compose/django/entrypoint.sh @@ -4,7 +4,7 @@ set -e # Since docker-compose relies heavily on environment variables itself for configuration, we'd have to define multiple # environment variables just to support cookiecutter out of the box. That makes no sense, so this little entrypoint # does all this for us. -export REDIS_URL=redis://redis:6379/0 +export CACHE_URL=redis://redis:6379/0 # the official postgres image uses 'postgres' as default user if not set explictly. if [ -z "$POSTGRES_ENV_POSTGRES_USER" ]; then @@ -13,7 +13,7 @@ fi export DATABASE_URL=postgres://$POSTGRES_ENV_POSTGRES_USER:$POSTGRES_ENV_POSTGRES_PASSWORD@postgres:5432/$POSTGRES_ENV_POSTGRES_USER -export CELERY_BROKER_URL=$REDIS_URL +export CELERY_BROKER_URL=$CACHE_URL # we copy the frontend files, if any so we can serve them from the outside if [ -d "frontend" ]; then diff --git a/api/config/settings/common.py b/api/config/settings/common.py index 3f7cc7503..8f00c02cc 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -123,7 +123,7 @@ MANAGERS = ADMINS # See: https://docs.djangoproject.com/en/dev/ref/settings/#databases DATABASES = { # Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ - 'default': env.db("DATABASE_URL", default="postgresql://postgres@postgres/postgres"), + 'default': env.db("DATABASE_URL"), } DATABASES['default']['ATOMIC_REQUESTS'] = True # @@ -198,7 +198,7 @@ CRISPY_TEMPLATE_PACK = 'bootstrap3' # STATIC FILE CONFIGURATION # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root -STATIC_ROOT = str(ROOT_DIR('staticfiles')) +STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles'))) # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url STATIC_URL = env("STATIC_URL", default='/staticfiles/') @@ -217,12 +217,10 @@ STATICFILES_FINDERS = ( # MEDIA CONFIGURATION # ------------------------------------------------------------------------------ # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root -MEDIA_ROOT = str(APPS_DIR('media')) - - +MEDIA_ROOT = env("MEDIA_ROOT", default=str(APPS_DIR('media'))) # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url -MEDIA_URL = '/media/' +MEDIA_URL = env("MEDIA_URL", default='/media/') # URL Configuration # ------------------------------------------------------------------------------ @@ -252,26 +250,24 @@ LOGIN_URL = 'account_login' # SLUGLIFIER AUTOSLUG_SLUGIFY_FUNCTION = 'slugify.slugify' -########## CELERY -INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',) -# if you are not using the django database broker (e.g. rabbitmq, redis, memcached), you can remove the next line. -INSTALLED_APPS += ('kombu.transport.django',) -BROKER_URL = env("CELERY_BROKER_URL", default='django://') -########## END CELERY - - +CACHE_DEFAULT = "redis://127.0.0.1:6379/0" CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0), - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "IGNORE_EXCEPTIONS": True, # mimics memcache behavior. - # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior - } - } + "default": env.cache_url('CACHE_URL', default=CACHE_DEFAULT) } +CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache" +CACHES["default"]["OPTIONS"] = { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "IGNORE_EXCEPTIONS": True, # mimics memcache behavior. + # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior +} + + +########## CELERY +INSTALLED_APPS += ('funkwhale_api.taskapp.celery.CeleryConfig',) +BROKER_URL = env( + "CELERY_BROKER_URL", default=env('CACHE_URL', default=CACHE_DEFAULT)) +########## END CELERY # Location of root django.contrib.admin URL, use {% url 'admin:index' %} ADMIN_URL = r'^admin/' # Your common stuff: Below this line define 3rd party library settings @@ -334,3 +330,7 @@ MUSICBRAINZ_CACHE_DURATION = env.int( ) CACHALOT_ENABLED = env.bool('CACHALOT_ENABLED', default=True) + + +# Custom Admin URL, use {% url 'admin:index' %} +ADMIN_URL = env('DJANGO_ADMIN_URL', default='^api/admin/') diff --git a/api/config/settings/production.py b/api/config/settings/production.py index e8a05bd3b..a132076c7 100644 --- a/api/config/settings/production.py +++ b/api/config/settings/production.py @@ -54,7 +54,7 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') # ------------------------------------------------------------------------------ # Hosts/domain names that are valid for this site # See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts -ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['funkwhale.io']) +ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS') # END SITE CONFIGURATION INSTALLED_APPS += ("gunicorn", ) @@ -65,10 +65,6 @@ INSTALLED_APPS += ("gunicorn", ) # ------------------------ DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' -# URL that handles the media served from MEDIA_ROOT, used for managing -# stored files. -MEDIA_URL = '/media/' - # Static Assets # ------------------------ STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage' @@ -92,11 +88,6 @@ TEMPLATES[0]['OPTIONS']['loaders'] = [ 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]), ] -# DATABASE CONFIGURATION -# ------------------------------------------------------------------------------ -# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ -DATABASES['default'] = env.db("DATABASE_URL") - # CACHING # ------------------------------------------------------------------------------ # Heroku URL does not pass the DB number, so we parse it in @@ -151,7 +142,5 @@ LOGGING = { } } -# Custom Admin URL, use {% url 'admin:index' %} -ADMIN_URL = env('DJANGO_ADMIN_URL') # Your production stuff: Below this line define 3rd party library settings diff --git a/api/config/settings/test.py b/api/config/settings/test.py index b8dd89b04..7d02c417b 100644 --- a/api/config/settings/test.py +++ b/api/config/settings/test.py @@ -22,6 +22,9 @@ CACHES = { 'LOCATION': '' } } +INSTALLED_APPS += ('kombu.transport.django',) +BROKER_URL = 'django://' + # TESTING # ------------------------------------------------------------------------------ TEST_RUNNER = 'django.test.runner.DiscoverRunner' diff --git a/api/funkwhale_api/providers/audiofile/management/commands/import_files.py b/api/funkwhale_api/providers/audiofile/management/commands/import_files.py index a34e36b4b..a4bfe555e 100644 --- a/api/funkwhale_api/providers/audiofile/management/commands/import_files.py +++ b/api/funkwhale_api/providers/audiofile/management/commands/import_files.py @@ -1,6 +1,6 @@ import glob from django.core.management.base import BaseCommand, CommandError -from funkwhale_api.providers.audiofile import importer +from funkwhale_api.providers.audiofile import tasks class Command(BaseCommand): @@ -61,7 +61,7 @@ class Command(BaseCommand): for path in matching: self.stdout.write(message.format(path)) try: - importer.from_path(path) + tasks.from_path(path) except Exception as e: self.stdout.write('Error: {}'.format(e)) diff --git a/api/funkwhale_api/providers/audiofile/importer.py b/api/funkwhale_api/providers/audiofile/tasks.py similarity index 100% rename from api/funkwhale_api/providers/audiofile/importer.py rename to api/funkwhale_api/providers/audiofile/tasks.py diff --git a/api/funkwhale_api/providers/audiofile/tests/test_disk_import.py b/api/funkwhale_api/providers/audiofile/tests/test_disk_import.py index 4a91a36eb..f8d36986a 100644 --- a/api/funkwhale_api/providers/audiofile/tests/test_disk_import.py +++ b/api/funkwhale_api/providers/audiofile/tests/test_disk_import.py @@ -3,7 +3,7 @@ import datetime import unittest from test_plus.test import TestCase -from funkwhale_api.providers.audiofile import importer +from funkwhale_api.providers.audiofile import tasks DATA_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -27,7 +27,7 @@ class TestAudioFile(TestCase): return_value='OggVorbis', ) with m1, m2: - track_file = importer.from_path( + track_file = tasks.from_path( os.path.join(DATA_DIR, 'dummy_file.ogg')) self.assertEqual( diff --git a/api/manage.py b/api/manage.py index 7b367ffeb..d99574ebe 100755 --- a/api/manage.py +++ b/api/manage.py @@ -2,8 +2,11 @@ import os import sys +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") from django.core.management import execute_from_command_line diff --git a/api/requirements.apt b/api/requirements.apt index 68eaf2881..86c7d8b20 100644 --- a/api/requirements.apt +++ b/api/requirements.apt @@ -1,46 +1,9 @@ -##basic build dependencies of various Django apps for Ubuntu 14.04 -#build-essential metapackage install: make, gcc, g++, build-essential -#required to translate gettext -#python-dev - -##shared dependencies of: -##Pillow, pylibmc zlib1g-dev - -##Postgresql and psycopg2 dependencies +libjpeg-dev +zlib1g-dev libpq-dev postgresql-client -##Pillow dependencies -#libtiff4-dev -#libjpeg8-dev -#libfreetype6-dev -#liblcms1-dev -#libwebp-dev - - -##django-extensions -#graphviz-dev - -##hitch -#python-setuptools -#python3-dev -#python-virtualenv -#python-pip -#firefox -#automake -#libtool -#libreadline6 -#libreadline6-dev -#libreadline-dev -libsqlite3-dev -#libxml2 -#libxml2-dev -#libssl-dev -#libbz2-dev -#wget -#curl -#llvm - libav-tools +python3-dev diff --git a/api/test.yml b/api/test.yml index 6215e27de..bd3a98e45 100644 --- a/api/test.yml +++ b/api/test.yml @@ -4,3 +4,5 @@ test: command: pytest volumes: - .:/app + environment: + - "DATABASE_URL=sqlite://" diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index fd3b4327f..9cbe278e8 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -1,13 +1,21 @@ -# If you're tweaking this file from the template, ensure you edit at lest the +# If you're tweaking this file from the template, ensure you edit at least the # following variables: # - DJANGO_SECRET_KEY # - DJANGO_ALLOWED_HOSTS +# Additionaly, on non-docker setup, you'll also have to tweak/uncomment those +# variables: +# - DATABASE_URL +# - CACHE_URL +# - STATIC_ROOT +# - MEDIA_ROOT + # Docker only # ----------- # The tag of the image we should use # (it will be interpolated in docker-compose file) +# You can comment or ignore this if you're not using docker FUNKWHALE_VERSION=latest @@ -17,11 +25,41 @@ FUNKWHALE_VERSION=latest # Set this variables to bind the API server to another interface/port # example: FUNKWHALE_API_IP=0.0.0.0 # example: FUNKWHALE_API_PORT=5678 -FUNKWHALE_API_IP= -FUNKWHALE_API_PORT= +FUNKWHALE_API_IP=127.0.0.1 +FUNKWHALE_API_PORT=5000 + # API/Django configuration +# Database configuration +# Examples: +# DATABASE_URL=postgresql://:@:/ +# DATABASE_URL=postgresql://funkwhale:passw0rd@localhost:5432/funkwhale_database +# Use the next one if you followed Debian installation guide +# DATABASE_URL=postgresql://funkwhale@:5432/funkwhale + +# Cache configuration +# Examples: +# CACHE_URL=redis://:/ +# CACHE_URL=redis://localhost:6379/0 +# Use the next one if you followed Debian installation guide +# CACHE_URL=redis://127.0.0.1:6379/0 + +# Where media files (such as album covers or audio tracks) should be stored +# on your system? +# (Ensure this directory actually exists) +# MEDIA_ROOT=/srv/funkwhale/data/media + +# Where static files (such as API css or icons) should be compiled +# on your system? +# (Ensure this directory actually exists) +# STATIC_ROOT=/srv/funkwhale/data/static + +# Update it to match the domain that will be used to reach your funkwhale +# instance +# Example: DJANGO_ALLOWED_HOSTS=funkwhale.yourdomain.com +DJANGO_ALLOWED_HOSTS=yourdomain + # which settings module should django use? # You don't have to touch this unless you really know what you're doing DJANGO_SETTINGS_MODULE=config.settings.production @@ -29,13 +67,9 @@ DJANGO_SETTINGS_MODULE=config.settings.production # Generate one using `openssl rand -base64 45`, for example DJANGO_SECRET_KEY= -# You don't have to edit this -DJANGO_ADMIN_URL=^api/admin/ - -# Update it to match the domain that will be used to reach your funkwhale -# instance -# Example: DJANGO_ALLOWED_HOSTS=funkwhale.yourdomain.com -DJANGO_ALLOWED_HOSTS=yourdomain +# You don't have to edit this, but you can put the admin on another URL if you +# want to +# DJANGO_ADMIN_URL=^api/admin/ # If True, unauthenticated users won't be able to query the API API_AUTHENTICATION_REQUIRED=True diff --git a/deploy/funkwhale-server.service b/deploy/funkwhale-server.service new file mode 100644 index 000000000..7ef6e3897 --- /dev/null +++ b/deploy/funkwhale-server.service @@ -0,0 +1,14 @@ +[Unit] +Description=Funkwhale application server +After=redis.service postgresql.service +PartOf=funkwhale.target + +[Service] +User=funkwhale +# adapt this depending on the path of your funkwhale installation +WorkingDirectory=/srv/funkwhale/api +EnvironmentFile=/srv/funkwhale/config/.env +ExecStart=/srv/funkwhale/virtualenv/bin/gunicorn config.wsgi:application -b ${FUNKWHALE_API_IP}:${FUNKWHALE_API_PORT} + +[Install] +WantedBy=multi-user.target diff --git a/deploy/funkwhale-worker.service b/deploy/funkwhale-worker.service new file mode 100644 index 000000000..2a25c2a1b --- /dev/null +++ b/deploy/funkwhale-worker.service @@ -0,0 +1,14 @@ +[Unit] +Description=Funkwhale celery worker +After=redis.service postgresql.service +PartOf=funkwhale.target + +[Service] +User=funkwhale +# adapt this depending on the path of your funkwhale installation +WorkingDirectory=/srv/funkwhale/api +EnvironmentFile=/srv/funkwhale/config/.env +ExecStart=/srv/funkwhale/virtualenv/bin/python manage.py celery worker + +[Install] +WantedBy=multi-user.target diff --git a/deploy/funkwhale.target b/deploy/funkwhale.target new file mode 100644 index 000000000..a920c7e34 --- /dev/null +++ b/deploy/funkwhale.target @@ -0,0 +1,3 @@ +[Unit] +Description=Funkwhale +Wants=funkwhale-server.service funkwhale-worker.service diff --git a/deploy/nginx.conf b/deploy/nginx.conf index a85230ae8..90b0f000e 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -1,29 +1,39 @@ +# Ensure you update at least the server_name variables to match your own +# domain + upstream funkwhale-api { # depending on your setup, you may want to udpate this server localhost:5000; } server { - listen 80; - listen [::]:80; - server_name demo.funkwhale.audio; - # useful for Let's Encrypt - location /.well-known/acme-challenge/ { allow all; } - location / { return 301 https://$host$request_uri; } + listen 80; + listen [::]:80; + # update this to match your instance name + server_name demo.funkwhale.audio; + # useful for Let's Encrypt + location /.well-known/acme-challenge/ { allow all; } + location / { return 301 https://$host$request_uri; } } server { listen 443 ssl http2; listen [::]:443 ssl http2; + # update this to match your instance name server_name demo.funkwhale.audio; # TLS + # Feel free to use your own configuration for SSL here or simply remove the + # lines and move the configuration to the previous server block if you + # don't want to run funkwhale behind https (this is not recommanded) + # have a look here for let's encrypt configuration: + # https://certbot.eff.org/all-instructions/#debian-9-stretch-nginx ssl_protocols TLSv1.2; ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA; ssl_prefer_server_ciphers on; ssl_session_cache shared:SSL:10m; - ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; + ssl_certificate /etc/letsencrypt/live/demo.funkwhale.audio/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/demo.funkwhale.audio/privkey.pem; # HSTS add_header Strict-Transport-Security "max-age=31536000"; diff --git a/dev.yml b/dev.yml index 78bf76fcd..54bc62d87 100644 --- a/dev.yml +++ b/dev.yml @@ -34,6 +34,8 @@ services: command: python manage.py celery worker environment: - C_FORCE_ROOT=true + - "DATABASE_URL=postgresql://postgres@postgres/postgres" + - "CACHE_URL=redis://redis:6379/0" volumes: - ./api:/app - ./data/music:/music @@ -46,12 +48,14 @@ services: volumes: - ./api:/app - ./data/music:/music + environment: + - "DATABASE_URL=postgresql://postgres@postgres/postgres" + - "CACHE_URL=redis://redis:6379/0" ports: - "12081:12081" links: - postgres - redis - - celeryworker nginx: env_file: .env.dev diff --git a/docs/changelog.rst b/docs/changelog.rst index 47bf0ed9c..0e6872f4b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,9 +21,7 @@ Changelog * [feature] can now import artist and releases from youtube and musicbrainz. This requires a YouTube API key for the search * [breaking] we now check for user permission before serving audio files, which requires -a specific configuration block in your reverse proxy configuration: - -.. code-block:: + a specific configuration block in your reverse proxy configuration:: location /_protected/media { internal; diff --git a/docs/conf.py b/docs/conf.py index 5a5effa3c..3a0c8f6f1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,10 +17,12 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('../api')) + +import funkwhale_api # NOQA # -- General configuration ------------------------------------------------ @@ -55,9 +57,11 @@ author = 'Eliot Berriot' # built documents. # # The short X.Y version. -version = '0.1' +# version = funkwhale_api.__version__ +# @TODO use real version here +version = 'feature/22-debian-installation' # The full version, including alpha/beta/rc tags. -release = '0.1' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -152,6 +156,3 @@ texinfo_documents = [ author, 'funkwhale', 'One line description of project.', 'Miscellaneous'), ] - - - diff --git a/docs/importing-music.rst b/docs/importing-music.rst index 5fdff7e93..3fa9e9990 100644 --- a/docs/importing-music.rst +++ b/docs/importing-music.rst @@ -5,10 +5,18 @@ From music directory on the server ---------------------------------- You can import music files in funkwhale assuming they are located on the server -and readable by the funkwhale application. +and readable by the funkwhale application. Your music files should contain at +least an ``artist``, ``album`` and ``title`` tags. -Assuming your music is located at ``/music`` and your music files contains at -least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as follows: +You can import those tracks as follows, assuming they are located in +``/srv/funkwhale/data/music``: + +.. code-block:: bash + + python api/manage.py import_files "/srv/funkwhale/data/music/**/*.ogg" --recursive --noinput + +When you use docker, the ``/srv/funkwhale/data/music`` is mounted from the host +to the ``/music`` directory on the container: .. code-block:: bash @@ -17,6 +25,7 @@ least an ``artist``, ``album`` and ``title`` tag, you can import those tracks as For the best results, we recommand tagging your music collection through `Picard `_ in order to have the best quality metadata. + .. note:: This command is idempotent, meaning you can run it multiple times on the same @@ -26,6 +35,18 @@ For the best results, we recommand tagging your music collection through At the moment, only OGG/Vorbis and MP3 files with ID3 tags are supported +.. note:: + + The --recursive flag will work only on Python 3.5+, which is the default + version When using Docker or Debian 9. If you use an older version of Python, + remove the --recursive flag and use more explicit import patterns instead:: + + # this will only import ogg files at the second level + "/srv/funkwhale/data/music/*/*.ogg" + # this will only import ogg files in the fiven directory + "/srv/funkwhale/data/music/System-of-a-down/*.ogg" + + Getting demo tracks ^^^^^^^^^^^^^^^^^^^ @@ -34,10 +55,10 @@ If you do not have any music on your server but still want to test the import process, you can call the following methods do download a few albums licenced under creative commons (courtesy of Jamendo): -.. code-block:: bash +.. parsed-literal:: - curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/download-tracks.sh" - curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/demo/music.txt" + curl -L -o download-tracks.sh "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/download-tracks.sh" + curl -L -o music.txt "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/demo/music.txt" chmod +x download-tracks.sh ./download-tracks.sh music.txt diff --git a/docs/installation/debian.rst b/docs/installation/debian.rst new file mode 100644 index 000000000..86ccb4dd3 --- /dev/null +++ b/docs/installation/debian.rst @@ -0,0 +1,280 @@ +Debian installation +=================== + +.. note:: + + this guide targets Debian 9, which is the latest debian, but should work + similarly on Debian 8. + +External dependencies +--------------------- + +The guides will focus on installing funkwhale-specific components and +dependencies. However, funkwhale requires a +:doc:`few external dependencies <./external_dependencies>` for which +documentation is outside of this document scope. + +Install utilities +----------------- + +You'll need a few utilities during this guide that are not always present by +default on system. You can install them using: + +.. code-block:: shell + + sudo apt-get update + sudo apt-get install curl python3-venv git unzip + + +Layout +------- + +All funkwhale-related files will be located under ``/srv/funkwhale`` apart +from database files and a few configuration files. We will also have a +dedicated ``funwhale`` user to launch the processes we need and own those files. + +You are free to use different values here, just remember to adapt those in the +next steps. + +.. _create-funkwhale-user: + +Create the user and the directory: + +.. code-block:: shell + + sudo adduser --system --home /srv/funkwhale funkwhale + cd /srv/funkwhale + +Log in as the newly created user from now on: + +.. code-block:: shell + + sudo -u funkwhale -H bash + +Now let's setup our directory layout. Here is how it will look like:: + + . + ├── config # config / environment files + ├── api # api code of your instance + ├── data # persistent data, such as music files + ├── front # frontend files for the web user interface + └── virtualenv # python dependencies for funkwhale + +Create the aforementionned directories: + +.. code-block:: shell + + mkdir -p config api data/static data/media data/music front + +The ``virtualenv`` directory is a bit special and will be created separately. + +Download latest funkwhale release +---------------------------------- + +Funkwhale is splitted in two components: + +1. The API, which will handle music storage and user accounts +2. The frontend, that will simply connect to the API to interact with its data + +Those components are packaged in subsequent releases, such as 0.1, 0.2, etc. +You can browse the :doc:`changelog ` for a list of available releases +and pick the one you want to install, usually the latest one should be okay. + +In this guide, we'll assume you want to install the latest version of funkwhale, +which is |version|: + +First, we'll download the latest api release. + +.. parsed-literal:: + + curl -L -o "api-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_api" + unzip "api-|version|.zip" -d extracted + mv extracted/api api + rmdir extracted + + +Then we'll download the frontend files: + +.. parsed-literal:: + + curl -L -o "front-|version|.zip" "https://code.eliotberriot.com/funkwhale/funkwhale/-/jobs/artifacts/|version|/download?job=build_front" + unzip "front-|version|.zip" -d extracted + mv extracted/front . + rmdir extracted + +You can leave the ZIP archives in the directory, this will help you know +which version you've installed next time you want to upgrade your installation. + +System dependencies +------------------- + +First, switch to the api directory: + +.. code-block:: shell + + cd api + +A few OS packages are required in order to run funkwhale. The list is available +in ``api/requirements.apt`` or by running +``./install_os_dependencies.sh list``. + +.. note:: + + Ensure you are running the next commands as root or using sudo + (and not as the funkwhale) user. + +You can install those packages all at once: + +.. code-block:: shell + + ./install_os_dependencies.sh install + +From now on you can switch back to the funkwhale user. + +Python dependencies +-------------------- + +Go back to the base directory: + +.. code-block:: shell + + cd /srv/funkwhale + +To avoid collisions with other software on your system, Python dependencies +will be installed in a dedicated +`virtualenv `_. + +First, create the virtualenv: + +.. code-block:: shell + + python3 -m venv /srv/funkwhale/virtualenv + +This will result in a ``virtualenv`` directory being created in +``/srv/funkwhale/virtualenv``. + +In the rest of this guide, we'll need to activate this environment to ensure +dependencies are installed within it, and not directly on your host system. + +This is done with the following command: + +.. code-block:: shell + + source /srv/funkwhale/virtualenv/bin/activate + +Finally, install the python dependencies: + +.. code-block:: shell + + pip install wheel + pip install -r api/requirements.txt + +.. important:: + + further commands involving python should always be run after you activated + the virtualenv, as described earlier, otherwise those commands will raise + errors + + +Environment file +---------------- + +You can now start to configure funkwhale. The main way to achieve that is by +adding an environment file that will host settings that are relevant to your +installation. + +Download the sample environment file: + +.. parsed-literal:: + + curl -L -o config/.env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/env.prod.sample" + +You can then edit it: the file is heavily commented, and the most relevant +configuration options are mentionned at the top of the file. + +Especially, populate the ``DATABASE_URL`` and ``CACHE_URL`` values based on +how you configured your PostgreSQL and Redis servers in +:doc:`external dependencies <./external_dependencies>`. + + +When you want to run command on the API server, such as to create the +database or compile static files, you have to ensure you source +the environment variables. + +This can be done like this:: + + export $(cat config/.env | grep -v ^# | xargs) + +The easier thing to do is to store this in a script:: + + cat > /srv/funkwhale/load_env <<'EOL' + #!/bin/bash + export $(cat /srv/funkwhale/config/.env | grep -v ^# | xargs) + EOL + chmod +x /srv/funkwhale/load_env + +You should now be able to run the following to populate your environment +variables easily: + +.. code-block:: shell + + source /srv/funkwhale/load_env + +.. note:: + + Remember to source ``load_env`` whenever you edit your .env file. + +Database setup +--------------- + +You should now be able to import the initial database structure: + +.. code-block:: shell + + python api/manage.py migrate + +This will create the required tables and rows. + +.. note:: + + You can safely execute this command any time you want, this will only + run unapplied migrations. + + +Create an admin account +----------------------- + +You can then create your first user account: + +.. code-block:: shell + + python api/manage.py createsuperuser + +If you ever want to change a user's password from the command line, just run: + +.. code-block:: shell + + python api/manage.py changepassword + +Collect static files +-------------------- + +Static files are the static assets used by the API server (icon PNGs, CSS, etc.). +We need to collect them explicitely, so they can be served by the webserver: + +.. code-block:: shell + + python api/manage.py collectstatic + +This should populate the directory you choose for the ``STATIC_ROOT`` variable +in your ``.env`` file. + +Systemd unit file +------------------ + +See :doc:`./systemd`. + +Reverse proxy setup +-------------------- + +See :ref:`reverse-proxy `. diff --git a/docs/installation/docker.rst b/docs/installation/docker.rst index 76958fb0b..34e8187c5 100644 --- a/docs/installation/docker.rst +++ b/docs/installation/docker.rst @@ -7,17 +7,17 @@ First, ensure you have `Docker `_ Download the sample docker-compose file: -.. code-block:: bash +.. parsed-literal:: mkdir -p /srv/funkwhale cd /srv/funkwhale - curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/deploy/docker-compose.yml" + curl -L -o docker-compose.yml "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/docker-compose.yml" Create your env file: -.. code-block:: bash +.. parsed-literal:: - curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/deploy/env.prod.sample" + curl -L -o .env "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/env.prod.sample" Ensure to edit it to match your needs (this file is heavily commented) diff --git a/docs/installation/external_dependencies.rst b/docs/installation/external_dependencies.rst new file mode 100644 index 000000000..fa0908545 --- /dev/null +++ b/docs/installation/external_dependencies.rst @@ -0,0 +1,62 @@ +External dependencies +===================== + + +.. note:: + + Those dependencies are handled automatically if you are + :doc:`deploying using docker <./docker>` + +Database setup (PostgreSQL) +--------------------------- + +Funkwhale requires a PostgreSQL database to work properly. Please refer +to the `PostgreSQL documentation `_ +for installation instructions specific to your os. + +On debian-like systems, you would install the database server like this: + +.. code-block:: shell + + sudo apt-get install postgresql + +The remaining steps are heavily inspired from `this Digital Ocean guide `_. + +Open a database shell: + +.. code-block:: shell + + sudo -u postgres psql + +Create the project database and user: + +.. code-block:: shell + + CREATE DATABASE funkwhale; + CREATE USER funkwhale; + GRANT ALL PRIVILEGES ON DATABASE funkwhale TO funkwhale; + +Assuming you already have :ref:`created your funkwhale user `, +you should now be able to open a postgresql shell: + +.. code-block:: shell + + sudo -u funkwhale -H psql + +Cache setup (Redis) +------------------- + +Funkwhale also requires a cache server: + +- To make the whole system faster, by caching network requests or database + queries +- To handle asynchronous tasks such as music import + +On debian-like distributions, a redis package is available, and you can +install it: + +.. code-block:: shell + + sudo apt-get install redis-server + +This should be enough to have your redis server set up. diff --git a/docs/installation/index.rst b/docs/installation/index.rst index 1544dfbf0..218049dd1 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -18,7 +18,10 @@ Available installation methods .. toctree:: :maxdepth: 1 + external_dependencies + debian docker + systemd .. _frontend-setup: @@ -33,10 +36,10 @@ Frontend setup Files for the web frontend are purely static and can simply be downloaded, unzipped and served from any webserver: -.. code-block:: bash +.. parsed-literal:: cd /srv/funkwhale - curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/builds/artifacts/master/download?job=build_front" + curl -L -o front.zip "https://code.eliotberriot.com/funkwhale/funkwhale/builds/artifacts/|version|/download?job=build_front" unzip front.zip .. _reverse-proxy-setup: @@ -58,8 +61,8 @@ Ensure you have a recent version of nginx on your server. On debian-like system, Then, download our sample virtualhost file: -.. code-block:: bash +.. parsed-literal:: - curl -L -o /etc/nginx/sites-enabled/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/master/deploy/nginx.conf" + curl -L -o /etc/nginx/sites-enabled/funkwhale.conf "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/nginx.conf" Ensure static assets and proxy pass match your configuration, and check the configuration is valid with ``nginx -t``. If everything is fine, you can restart your nginx server with ``service nginx restart``. diff --git a/docs/installation/systemd.rst b/docs/installation/systemd.rst new file mode 100644 index 000000000..67af98432 --- /dev/null +++ b/docs/installation/systemd.rst @@ -0,0 +1,42 @@ +Systemd configuration +---------------------- + +Systemd offers a convenient way to manage your funkwhale instance if you're +not using docker. + +We'll see how to setup systemd to proprely start a funkwhale instance. + +First, download the sample unitfiles: + +.. parsed-literal:: + + curl -L -o "/etc/systemd/system/funkwhale.target" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale.target" + curl -L -o "/etc/systemd/system/funkwhale-server.service" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale-server.service" + curl -L -o "/etc/systemd/system/funkwhale-worker.service" "https://code.eliotberriot.com/funkwhale/funkwhale/raw/|version|/deploy/funkwhale-worker.service" + +This will download three unitfiles: + +- ``funkwhale-server.service`` to launch the funkwhale web server +- ``funkwhale-worker.service`` to launch the funkwhale task worker +- ``funkwhale.target`` to easily stop and start all of the services at once + +You can of course review and edit them to suit your deployment scenario +if needed, but the defaults should be fine. + +Once the files are downloaded, reload systemd: + +.. code-block:: shell + + systemctl daemon-reload + +And start the services: + +.. code-block:: shell + + systemctl start funkwhale.target + +You can check the statuses of all processes like this: + +.. code-block:: shell + + systemctl status funkwhale-\*