Add support for debugging and testing python in gitpod

.gitignore

@@ -95,4 +95,4 @@ _build
 front/src/translations.json
 front/src/translations/*.json
 front/locales/en_US/LC_MESSAGES/app.po
-*.prof
+*.prof

.gitpod.yml

@@ -1,67 +1,49 @@
+image:
+  file: .gitpod/Dockerfile
+
 tasks:
-  - name: Docker
+  - name: Backend
     env:
-      COMPOSE_FILE: dev.yml
+      ENV_FILE: /workspace/funkwhale/.gitpod/.env
+      COMPOSE_FILE: /workspace/funkwhale/.gitpod/docker-compose.yml
+    before: |
+      cp .gitpod/gitpod.env .gitpod/.env
+      cd api
     init: |
-      # Install frontend depencencies locally
-      cd front
-      yarn install
-      cd ..
+      mkdir -p ../data/media/attachments ../data/music ../data/staticfiles
+      docker-compose up -d
 
-      # Prepare prebuild .env
-      echo "# Gitpod Environment Variables" > .env
+      poetry env use python
+      poetry install
 
-      # Prepare docker
-      docker network create federation
-      docker-compose pull
-      docker-compose build
-      docker-compose up -d postgres redis
-      sleep 10 # allow postgres and redis to initialize
+      gp ports await 5432
 
-      # Prepare backend
-      docker-compose run --rm api python manage.py migrate
-      docker-compose run --rm api python manage.py createsuperuser --no-input --username gitpod --email gitpod@example.com
-      docker-compose run --rm api python manage.py fw users set --password "gitpod" gitpod --no-input
-
-      # Compile frontend locales
-      docker-compose run --rm front yarn run i18n-compile
-
-      # Start API to let script create an actor
-      docker-compose up -d nginx
-      gp ports await 8000
-
-      # Clone music repo
-      git clone https://dev.funkwhale.audio/funkwhale/catalog.git
-      sudo mv catalog/music data
-      sudo chown -R root:root data/music
-      rm -rf catalog
-
-      # Login with cURL to create actor
-      python .gitpod/init_actor.py
-
-      # Import music
-      docker-compose down
-      LIBRARY_ID=`cat .gitpod/create_library.py | docker-compose run --rm -T api python manage.py shell -i python`
-      docker-compose run --rm api python manage.py import_files $LIBRARY_ID "/music/" --recursive --noinput --in-place
-
-      # Stop docker
-      docker-compose stop
+      poetry run python manage.py migrate
+      poetry run python manage.py gitpod init
     command: |
-      # Prepare workspace .env
-      echo "MEDIA_URL=`gp url 8000`/media/" >> .env
-      echo "STATIC_URL=`gp url 8000`/staticfiles/" >> .env
-      echo "FUNKWHALE_HOSTNAME=`gp url 8000 | sed 's#https://##'`" >> .env
-      echo "FUNKWHALE_PROTOCOL=https" >> .env
-      echo "GITPOD_WORKSPACE_URL=$GITPOD_WORKSPACE_URL" >> .env
-      echo "HMR_PORT=8000" >> .env
-      echo "VUE_APP_INSTANCE_URL=$VUE_APP_INSTANCE_URL" >> .env
+      echo "MEDIA_URL=`gp url 8000`/media/" >> ../.gitpod/.env
+      echo "STATIC_URL=`gp url 8000`/staticfiles/" >> ../.gitpod/.env
+      echo "FUNKWHALE_HOSTNAME=`gp url 8000 | sed 's#https://##'`" >> ../.gitpod/.env
+      echo "FUNKWHALE_PROTOCOL=https" >> ../.gitpod/.env
 
-      # Start app
-      docker-compose up front api nginx
+      docker-compose up -d
+      gp ports await 5432
+      poetry run python manage.py collectstatic --no-input
+      poetry run python manage.py gitpod dev
+
+  - name: Frontend
+    env:
+      HMR_PORT: 8000
+    before: cd front
+    init: |
+      yarn install
+      yarn run i18n-compile
+    command: yarn dev --host 0.0.0.0 --base /front/
 
   - name: Welcome to Funkwhale development!
     env:
-      COMPOSE_FILE: dev.yml
+      COMPOSE_FILE: /workspace/funkwhale/.gitpod/docker-compose.yml
+      ENV_FILE: /workspace/funkwhale/.gitpod/.env
     command: |
       clear
       echo ""
@@ -78,12 +60,33 @@ ports:
     visibility: public
     onOpen: notify
 
+  - port: 5000
+    visibility: private
+    onOpen: ignore
+
+  - port: 5432
+    visibility: private
+    onOpen: ignore
+
+  - port: 5678
+    visibility: private
+    onOpen: ignore
+
+  - port: 6379
+    visibility: private
+    onOpen: ignore
+
+  - port: 8080
+    visibility: private
+    onOpen: ignore
+
 vscode:
   extensions:
     - lukashass.volar
-    - lextudio.restructuredtext
-    - trond-snekvik.simple-rst
     - ms-python.python
     - ms-toolsai.jupyter
     - ms-toolsai.jupyter-keymap
     - ms-toolsai.jupyter-renderers
+    - hbenl.vscode-test-explorer
+    - hbenl.test-adapter-converter
+    - littlefoxteam.vscode-python-test-adapter

.gitpod/Dockerfile

@@ -0,0 +1,9 @@
+FROM gitpod/workspace-full
+USER gitpod
+
+RUN sudo apt update -y \
+    && sudo apt install libsasl2-dev libldap2-dev libssl-dev ffmpeg -y
+
+RUN pip install poetry \
+    && poetry config virtualenvs.create true \
+    && poetry config virtualenvs.in-project true 

.gitpod/create_library.py

@@ -1,17 +0,0 @@
-from funkwhale_api.music.models import Library
-from django.contrib.auth import get_user_model
-
-actor = get_user_model().objects.get(username='gitpod').actor
-
-try:
-    library = Library.objects.get(actor=actor)
-except:
-    #  Create library
-    library = Library()
-    library.actor = actor
-    library.description = 'Libre music to build a starter catalog for your instance'
-    library.name = 'funkwhale/catalog'
-    library.privacy_level = 'everyone'
-    library.save()
-
-print(str(library.uuid))

.gitpod/docker-compose.yml

@@ -0,0 +1,43 @@
+version: '3'
+
+services:
+  postgres:
+    image: postgres:14-alpine
+    environment:
+      - "POSTGRES_HOST_AUTH_METHOD=trust"
+    volumes:
+      - "../data/postgres:/var/lib/postgresql/data"
+    ports:
+      - 5432:5432
+
+  redis:
+    image: redis:7-alpine
+    volumes:
+      - "../data/redis:/data"
+    ports:
+      - 6379:6379
+
+  nginx:
+    command: /entrypoint.sh
+    env_file:
+      - ./.env
+    image: nginx
+    ports:
+      - 8000:80
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    environment:
+      - "NGINX_MAX_BODY_SIZE=100M"
+      - "FUNKWHALE_API_IP=host.docker.internal"
+      - "FUNKWHALE_API_PORT=5000"
+      - "FUNKWHALE_FRONT_IP=host.docker.internal"
+      - "FUNKWHALE_FRONT_PORT=8080"
+      - "FUNKWHALE_HOSTNAME=${FUNKWHALE_HOSTNAME-host.docker.internal}"
+    volumes:
+      - ../data/media:/protected/media:ro
+      - ../data/music:/music:ro
+      - ../data/staticfiles:/staticfiles:ro
+      - ../deploy/funkwhale_proxy.conf:/etc/nginx/funkwhale_proxy.conf:ro
+      - ../docker/nginx/conf.dev:/etc/nginx/nginx.conf.template:ro
+      - ../docker/nginx/entrypoint.sh:/entrypoint.sh:ro
+      - ../front:/frontend:ro

.gitpod/gitpod.env

@@ -0,0 +1,25 @@
+# Dev Environment Variables
+DJANGO_ALLOWED_HOSTS=.funkwhale.test,localhost,nginx,0.0.0.0,127.0.0.1,.gitpod.io
+DJANGO_SETTINGS_MODULE=config.settings.local
+C_FORCE_ROOT=true
+BROWSABLE_API_ENABLED=True
+FORWARDED_PROTO=http
+LDAP_ENABLED=False
+FUNKWHALE_SPA_HTML_ROOT=http://localhost:8000/front/
+FUNKWHALE_URL=http://localhost:8000/
+MUSIC_DIRECTORY_PATH=/workspace/funkwhale/data/music
+STATIC_ROOT=/workspace/funkwhale/data/staticfiles/
+MEDIA_ROOT=/workspace/funkwhale/data/media/
+
+PYTHONTRACEMALLOC=0
+PYTHONDONTWRITEBYTECODE=true
+
+POSTGRES_VERSION=14
+DEBUG=true
+
+
+# Django Environment Variables
+DATABASE_URL=postgresql://postgres@localhost:5432/postgres
+DJANGO_SECRET_KEY=gitpod
+
+# Gitpod Environment Variables

.gitpod/init_actor.py

@@ -1,21 +0,0 @@
-import requests
-
-# Login to initialize user actor
-req = requests.Session()
-
-res = req.get('http://localhost:8000/login')
-print(res.status_code, res.cookies)
-token = res.cookies['csrftoken']
-
-res = req.post('http://localhost:8000/api/v1/users/login', data={
-    'username': 'gitpod',
-    'password': 'gitpod',
-    'csrfmiddlewaretoken': token,
-})
-print(res.status_code, res.content)
-
-res = req.get('http://localhost:8000/')
-print(res.status_code)
-
-if res.status_code == 401:
-    exit(1)

.vscode/launch.json

@@ -0,0 +1,34 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Attach python debugger",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "django": true
+        },
+        {
+            "name": "Debug python",
+            "type": "python",
+            "request": "launch",
+            "module": "uvicorn",
+            "cwd": "${workspaceFolder}/api",
+            "envFile": "${workspaceFolder}/.gitpod/.env",
+            "args": [
+                "--reload", "config.asgi:application",
+                "--host", "0.0.0.0",
+                "--port", "5000",
+                "--reload-dir", "config/",
+                "--reload-dir", "funkwhale_api/"
+            ],
+            "django": true
+        }
+    ]
+}

.vscode/settings.json

@@ -1,3 +1,11 @@
 {
-    "esbonio.sphinx.confDir": ""
+    "python.defaultInterpreterPath": "/workspace/funkwhale/api/.venv/bin/python",
+    "python.testing.cwd": "/workspace/funkwhale/api",
+    "python.envFile": "/workspace/funkwhale/.gitpod/.env",
+    "python.testing.pytestArgs": [
+        "--cov=funkwhale_api",
+        "tests/"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
 }

api/funkwhale_api/common/management/commands/gitpod.py

@@ -0,0 +1,72 @@
+from django.core.management.commands.migrate import Command as BaseCommand
+from django.core.management import call_command
+from funkwhale_api.music.models import Library
+from funkwhale_api.users.models import User
+import uvicorn
+import debugpy
+import os
+
+
+class Command(BaseCommand):
+    help = "Manage gitpod environment"
+
+    def add_arguments(self, parser):
+        parser.add_argument("command", nargs="?", type=str)
+
+    def handle(self, *args, **options):
+        command = options["command"]
+
+        if not command:
+            return self.show_help()
+
+        if command == "init":
+            return self.init()
+
+        if command == "dev":
+            return self.dev()
+
+    def show_help(self):
+        self.stdout.write("")
+        self.stdout.write("Available commands:")
+        self.stdout.write("init - Initialize gitpod workspace")
+        self.stdout.write("dev - Run Funkwhale in development mode with debug server")
+        self.stdout.write("")
+
+    def init(self):
+        try:
+            user = User.objects.get(username="gitpod")
+        except Exception:
+            call_command("createsuperuser", username="gitpod", email="gitpod@example.com", no_input=False)
+            user = User.objects.get(username="gitpod")
+
+        user.set_password('gitpod')
+        if not user.actor:
+            user.create_actor()
+
+        user.save()
+
+        # Download music catalog
+        os.system("git clone https://dev.funkwhale.audio/funkwhale/catalog.git /tmp/catalog")
+        os.system("mv -f /tmp/catalog/music /workspace/funkwhale/data")
+        os.system("rm -rf /tmp/catalog/music")
+
+        # # Import music catalog into library
+        call_command("script", "migrate_to_user_libraries", no_input=False)
+        call_command(
+            "import_files",
+            Library.objects.get(actor=user.actor).uuid,
+            "/workspace/funkwhale/data/music/",
+            recursive=True,
+            in_place=True,
+            no_input=False,
+        )
+
+    def dev(self):
+        debugpy.listen(5678)
+        uvicorn.run(
+            "config.asgi:application",
+            host="0.0.0.0",
+            port=5000,
+            reload=True,
+            reload_dirs=["/workspace/funkwhale/api/config/", "/workspace/funkwhale/api/funkwhale_api/"],
+        )

api/funkwhale_api/music/management/commands/import_files.py

@@ -279,7 +279,7 @@ class Command(BaseCommand):
                 if p and not import_path.startswith(p):
                     raise CommandError(
                         "Importing in-place only works if importing "
-                        "from {} (MUSIC_DIRECTORY_PATH), as this directory"
+                        "from {} (MUSIC_DIRECTORY_PATH), as this directory "
                         "needs to be accessible by the webserver."
                         "Culprit: {}".format(p, import_path)
                     )

api/poetry.lock

@@ -529,6 +529,14 @@ twisted = {version = ">=18.7", extras = ["tls"]}
 [package.extras]
 tests = ["hypothesis (==4.23)", "pytest (>=3.10,<4.0)", "pytest-asyncio (>=0.8,<1.0)"]
 
+[[package]]
+name = "debugpy"
+version = "1.6.2"
+description = "An implementation of the Debug Adapter Protocol for Python"
+category = "dev"
+optional = false
+python-versions = ">=3.7"
+
 [[package]]
 name = "decorator"
 version = "5.1.1"
@@ -2120,7 +2128,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "66a96848a355caf841c42228ca2f411fef64f2507cf91d6d056038b2d2a2c22b"
+content-hash = "f09ed385656e74fcd8d2c68ef9c43768de5459eed452dc9a4544df19894b7bfe"
 
 [metadata.files]
 aiohttp = [
@@ -2470,6 +2478,26 @@ daphne = [
     {file = "daphne-3.0.2-py3-none-any.whl", hash = "sha256:a9af943c79717bc52fe64a3c236ae5d3adccc8b5be19c881b442d2c3db233393"},
     {file = "daphne-3.0.2.tar.gz", hash = "sha256:76ffae916ba3aa66b46996c14fa713e46004788167a4873d647544e750e0e99f"},
 ]
+debugpy = [
+    {file = "debugpy-1.6.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:77a47d596ce8c69673d5f0c9876a80cb5a6cbc964f3b31b2d44683c7c01b6634"},
+    {file = "debugpy-1.6.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:726e5cc0ed5bc63e821dc371d88ddae5cba85e2ad207bf5fefc808b29421cb4c"},
+    {file = "debugpy-1.6.2-cp310-cp310-win32.whl", hash = "sha256:9809bd1cdc0026fab711e280e0cb5d8f89ae5f4f74701aba5bda9a20a6afb567"},
+    {file = "debugpy-1.6.2-cp310-cp310-win_amd64.whl", hash = "sha256:40741d4bbf59baca1e97a5123514afcc036423caae5f24db23a865c0b4167c34"},
+    {file = "debugpy-1.6.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:67749e972213c395647a8798cc8377646e581e1fe97d0b1b7607e6b112ae4511"},
+    {file = "debugpy-1.6.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e3c43d650a1e5fa7110af380fb59061bcba1e7348c00237e7473c55ae499b96"},
+    {file = "debugpy-1.6.2-cp37-cp37m-win32.whl", hash = "sha256:9e572c2ac3dd93f3f1a038a9226e7cc0d7326b8d345c9b9ce6fbf9cb9822e314"},
+    {file = "debugpy-1.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:ac5d9e625d291a041ff3eaf65bdb816eb79a5b204cf9f1ffaf9617c0eadf96fa"},
+    {file = "debugpy-1.6.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:9f72435bc9a2026a35a41221beff853dd4b6b17567ba9b9d349ee9512eb71ce6"},
+    {file = "debugpy-1.6.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aaf579de5ecd02634d601d7cf5b6baae5f5bab89a55ef78e0904d766ef477729"},
+    {file = "debugpy-1.6.2-cp38-cp38-win32.whl", hash = "sha256:0984086a670f46c75b5046b39a55f34e4120bee78928ac4c3c7f1c7b8be1d8be"},
+    {file = "debugpy-1.6.2-cp38-cp38-win_amd64.whl", hash = "sha256:19337bb8ff87da2535ac00ea3877ceaf40ff3c681421d1a96ab4d67dad031a16"},
+    {file = "debugpy-1.6.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:163f282287ce68b00a51e9dcd7ad461ef288d740dcb3a2f22c01c62f31b62696"},
+    {file = "debugpy-1.6.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4909bb2f8e5c8fe33d6ec5b7764100b494289252ebe94ec7838b30467435f1cb"},
+    {file = "debugpy-1.6.2-cp39-cp39-win32.whl", hash = "sha256:3b4657d3cd20aa454b62a70040524d3e785efc9a8488d16cd0e6caeb7b2a3f07"},
+    {file = "debugpy-1.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:79d9ac34542b830a7954ab111ad8a4c790f1f836b895d03223aea4216b739208"},
+    {file = "debugpy-1.6.2-py2.py3-none-any.whl", hash = "sha256:0bfdcf261f97a603d7ef7ab6972cdf7136201fde93d19bf3f917d0d2e43a5694"},
+    {file = "debugpy-1.6.2.zip", hash = "sha256:e6047272e97a11aa6898138c1c88c8cf61838deeb2a4f0a74e63bb567f8dafc6"},
+]
 decorator = [
     {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
     {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},

api/pyproject.toml

@@ -57,7 +57,7 @@ django-auth-ldap = "==4.1.0"
 uvicorn = {version = "==0.17.6", extras = ["standard"]}
 django-cache-memoize = "0.1.10"
 requests-http-message-signatures = "==0.3.1"
-drf-spectacular = "0.22.1"
+drf-spectacular = "==0.22.1"
 
 [tool.poetry.dev-dependencies]
 flake8 = "==3.9.2"
@@ -80,6 +80,7 @@ aioresponses = "==0.7.3"
 prompt-toolkit = "==3.0.30"
 black = "==22.6.0"
 ipdb = "==0.13.9"
+debugpy = "==1.6.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

changes/changelog.d/1875.enchancement

@@ -0,0 +1 @@
+Add python debug and test support for gitpod