Implement several pipeline improvements

* Build docker image to run tests in
* Collect backend coverage reports
* Update dependencies

.gitlab-ci.yml

@@ -13,8 +13,8 @@ variables:
 stages:
   - review
   - lint
-  - test
   - build
+  - test
   - deploy
   - deps
 
@@ -123,7 +123,7 @@ test_api:
     - postgres:11
     - redis:5
   stage: test
-  image: funkwhale/funkwhale:develop
+  image: $CI_REGISTRY/$IMAGE
   cache:
     key: "$CI_PROJECT_ID__pip_cache"
     paths:
@@ -136,20 +136,21 @@ test_api:
   only:
     - branches
   before_script:
-    - apk add make git gcc python3-dev musl-dev
+    - apk add make git gcc python3-dev musl-dev python3-dev openssl-dev cargo
     - apk add postgresql-dev py3-psycopg2 libldap libffi-dev make zlib-dev jpeg-dev openldap-dev
     - cd api
     - pip3 install -r requirements/base.txt
     - pip3 install -r requirements/local.txt
     - pip3 install -r requirements/test.txt
   script:
-    - pytest --cov=funkwhale_api --junitxml=report.xml tests/
+    - pytest --cov-report xml --cov=funkwhale_api --junitxml=report.xml tests/
   tags:
     - docker
   artifacts:
     when: always
     reports:
       junit: api/report.xml
+      cobertura: api/coverage.xml
 
 test_front:
   interruptible: true
@@ -198,10 +199,21 @@ build_front:
     - tags@funkwhale/funkwhale
     - master@funkwhale/funkwhale
     - develop@funkwhale/funkwhale
-
   tags:
     - docker
 
+build_backend:
+  stage: build
+  image: bash
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+  script:
+    - cd api
+    - docker build -t $CI_REGISTRY/$IMAGE .
+    - docker push $CI_REGISTRY/$IMAGE
+  tags:
+    - docker-build
+
 pages:
   stage: test
   image: python:3.6
@@ -296,7 +308,7 @@ build_api:
 check_api_dependencies:
   interruptible: true
   stage: deps
-  image: funkwhale/funkwhale:develop
+  image: $CI_REGISTRY/$IMAGE
   cache:
     key: "$CI_PROJECT_ID__pip_cache"
     paths:

api/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.10 as builder
+FROM alpine:3.13 as builder
 
 RUN \
     echo 'installing dependencies' && \
@@ -15,6 +15,8 @@ RUN \
     zlib-dev           \
     jpeg-dev           \
     openldap-dev \
+    openssl-dev \
+    cargo \
     && \
     \
     ln -s /usr/bin/python3 /usr/bin/python
@@ -41,7 +43,7 @@ RUN \
     if [ "$install_dev_deps" = "1" ] ; then echo "Installing dev dependencies" && pip3 install --no-cache-dir -r /requirements/local.txt -r /requirements/test.txt ; else echo "Skipping dev deps installation" ; fi
 
 
-FROM alpine:3.10 as build-image
+FROM alpine:3.13 as build-image
 
 COPY --from=builder /venv /venv
 # emulate activation by prefixing PATH

api/funkwhale_api/federation/models.py

@@ -142,7 +142,6 @@ class Domain(models.Model):
         from funkwhale_api.music import models as music_models
 
         data = Domain.objects.filter(pk=self.pk).aggregate(
-            actors=models.Count("actors", distinct=True),
             outbox_activities=models.Count("actors__outbox_activities", distinct=True),
             libraries=models.Count("actors__libraries", distinct=True),
             channels=models.Count("actors__owned_channels", distinct=True),
@@ -152,6 +151,7 @@ class Domain(models.Model):
             emitted_library_follows=models.Count(
                 "actors__library_follows", distinct=True
             ),
+            actors=models.Count("actors", distinct=True),
         )
         data["artists"] = music_models.Artist.objects.filter(
             from_activity__actor__domain_id=self.pk
@@ -283,12 +283,12 @@ class Actor(models.Model):
 
         data = Actor.objects.filter(pk=self.pk).aggregate(
             outbox_activities=models.Count("outbox_activities", distinct=True),
-            libraries=models.Count("libraries", distinct=True),
             channels=models.Count("owned_channels", distinct=True),
             received_library_follows=models.Count(
                 "libraries__received_follows", distinct=True
             ),
             emitted_library_follows=models.Count("library_follows", distinct=True),
+            libraries=models.Count("libraries", distinct=True),
         )
         data["artists"] = music_models.Artist.objects.filter(
             from_activity__actor=self.pk

api/funkwhale_api/federation/tasks.py

@@ -461,7 +461,9 @@ class PreserveSomeDataCollector(Collector):
 
     def related_objects(self, related, *args, **kwargs):
         qs = super().related_objects(related, *args, **kwargs)
-        if related.name == "outbox_activities":
+        # We can only exclude the actions if these fields are available, most likely its a
+        # model.Activity than
+        if hasattr(related, "type") and hasattr(related, "creation_date"):
             # exclude the delete activity can be broadcasted properly
             qs = qs.exclude(type="Delete", creation_date__gte=self.creation_date)
 

api/requirements/base.txt

@@ -1,5 +1,5 @@
-django~=3.0.8
-setuptools>=49
+django~=3.2.3
+setuptools~=56.2.0
 # Configuration
 django-environ~=0.4.0
 
@@ -11,7 +11,7 @@ django-allauth~=0.42.0
 psycopg2-binary~=2.8.0
 
 # Time zones support
-pytz==2020.1
+pytz==2021.1
 
 # Redis support
 django-redis~=4.12.0
@@ -29,7 +29,7 @@ persisting-theory~=0.2.0
 django-versatileimagefield~=2.0.0
 django-filter~=2.4.0
 django-rest-auth~=0.9.0
-ipython~=7.22.0
+ipython~=7.23.1
 mutagen~=1.45.0
 
 pymemoize~=1.0.0
@@ -41,13 +41,13 @@ channels_redis~=3.2.0
 uvicorn[standard]~=0.13.4
 gunicorn~=20.1.0
 
-cryptography~=2.9.0
+cryptography~=3.4.7
 # requests-http-signature==0.0.3
 # clone until the branch is merged and released upstream
 git+https://github.com/agateblue/requests-http-signature.git@signature-header-support
 django-cleanup~=5.2.0
 requests~=2.25.1
-pyOpenSSL~=19.1.0
+pyOpenSSL~=20.0.1
 
 # for LDAP authentication
 python-ldap~=3.3.0
@@ -61,14 +61,14 @@ django-oauth-toolkit~=1.3.0
 django-storages~=1.11.1
 boto3~=1.17.59
 unicode-slugify~=0.1.0
-django-cacheops~=5.1.0
+django-cacheops~=6.0.0
 
+service_identity~=21.1.0
 click~=7.1.0
-service_identity~=18.1.0
 markdown~=3.3.4
 bleach~=3.3.0
 feedparser~=6.0.0
-watchdog~=2.0.3
+watchdog~=2.1.2
 
 ## Pin third party dependency to avoid issue with latest version
 twisted==20.3.0

api/requirements/local.txt

@@ -2,7 +2,7 @@
 
 coverage~=5.5.0
 django_coverage_plugin~=1.8.0
-factory_boy~=2.11.0
+factory_boy~=2.12.0
 
 # django-debug-toolbar that works with Django 1.5+
 django-debug-toolbar~=3.2.1

api/requirements/test.txt

@@ -2,11 +2,11 @@
 
 flake8~=3.9.1
 pytest~=6.2.3
-pytest-cov~=2.11.1
-pytest-django~=4.2.0
+pytest-cov~=2.12.0
+pytest-django~=4.3.0
 pytest-env~=0.6.0
 pytest-mock~=3.6.0
-pytest-randomly~=3.7.0
+pytest-randomly~=3.8.0
 pytest-sugar~=0.9.0
 requests-mock~=1.9.0
-faker~=8.1.1
+faker~=8.2.0