Merge branch 'master' into develop

.gitlab-ci.yml

@@ -224,7 +224,7 @@ docker_release:
   before_script:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
     - cp -r front/dist api/frontend
-    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
   script:
     - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
     - cd api
@@ -233,6 +233,7 @@ docker_release:
     - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi
   only:
     - develop@funkwhale/funkwhale
+    - master@funkwhale/funkwhale
     - tags@funkwhale/funkwhale
   tags:
     - docker-build
@@ -246,7 +247,7 @@ docker_all_in_one_release:
     BUILD_PATH: all_in_one
   before_script:
     - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD
-    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
   script:
     - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi
     - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip
@@ -261,6 +262,7 @@ docker_all_in_one_release:
     - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi
   only:
     - develop@funkwhale/funkwhale
+    - master@funkwhale/funkwhale
     - tags@funkwhale/funkwhale
   tags:
     - docker-build
@@ -275,7 +277,7 @@ build_api:
       - api
   script:
     - rm -rf api/tests
-    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
+    - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi);
     - chmod -R 750 api
     - echo Done!
   only:

api/funkwhale_api/federation/tasks.py

@@ -50,6 +50,7 @@ def clean_music_cache():
         )
         .local(False)
         .exclude(audio_file="")
+        .filter(Q(source__startswith="http://") | Q(source__startswith="https://"))
         .only("audio_file", "id")
         .order_by("id")
     )

api/funkwhale_api/music/tasks.py

@@ -817,9 +817,15 @@ def get_prunable_tracks(
     Returns a list of tracks with no associated uploads,
     excluding the one that were listened/favorited/included in playlists.
     """
-
+    purgeable_tracks_with_upload = (
+        models.Upload.objects.exclude(track=None)
+        .filter(import_status="skipped")
+        .values("track")
+    )
     queryset = models.Track.objects.all()
-    queryset = queryset.filter(uploads__isnull=True)
+    queryset = queryset.filter(
+        Q(uploads__isnull=True) | Q(pk__in=purgeable_tracks_with_upload)
+    )
     if exclude_favorites:
         queryset = queryset.filter(track_favorites__isnull=True)
     if exclude_playlists:

api/tests/federation/test_tasks.py

@@ -16,20 +16,28 @@ def test_clean_federation_music_cache_if_no_listen(preferences, factories):
     preferences["federation__music_cache_duration"] = 60
     remote_library = factories["music.Library"]()
     upload1 = factories["music.Upload"](
-        library=remote_library, accessed_date=timezone.now()
+        library=remote_library,
+        accessed_date=timezone.now(),
+        source="https://upload1.mp3",
     )
     upload2 = factories["music.Upload"](
         library=remote_library,
         accessed_date=timezone.now() - datetime.timedelta(minutes=61),
+        source="https://upload2.mp3",
+    )
+    upload3 = factories["music.Upload"](
+        library=remote_library, accessed_date=None, source="http://upload3.mp3"
     )
-    upload3 = factories["music.Upload"](library=remote_library, accessed_date=None)
     # local upload, should not be cleaned
     upload4 = factories["music.Upload"](library__actor__local=True, accessed_date=None)
+    # non-http source , not cleaned
+    upload5 = factories["music.Upload"](accessed_date=None, source="noop")
 
     path1 = upload1.audio_file_path
     path2 = upload2.audio_file_path
     path3 = upload3.audio_file_path
     path4 = upload4.audio_file_path
+    path5 = upload5.audio_file_path
 
     tasks.clean_music_cache()
 
@@ -37,15 +45,18 @@ def test_clean_federation_music_cache_if_no_listen(preferences, factories):
     upload2.refresh_from_db()
     upload3.refresh_from_db()
     upload4.refresh_from_db()
+    upload5.refresh_from_db()
 
     assert bool(upload1.audio_file) is True
     assert bool(upload2.audio_file) is False
     assert bool(upload3.audio_file) is False
     assert bool(upload4.audio_file) is True
+    assert bool(upload5.audio_file) is True
     assert os.path.exists(path1) is True
     assert os.path.exists(path2) is False
     assert os.path.exists(path3) is False
     assert os.path.exists(path4) is True
+    assert os.path.exists(path5) is True
 
 
 def test_clean_federation_music_cache_orphaned(settings, preferences, factories):

api/tests/music/test_tasks.py

@@ -867,6 +867,8 @@ def test_clean_transcoding_cache(preferences, now, factories):
 
 def test_get_prunable_tracks(factories):
     prunable_track = factories["music.Track"]()
+    # track is still prunable if it has a skipped upload linked to it
+    factories["music.Upload"](import_status="skipped", track=prunable_track)
     # non prunable tracks
     factories["music.Upload"]()
     factories["favorites.TrackFavorite"]()

changes/changelog.d/1011.bugfix

@@ -0,0 +1 @@
+Ensure tracks linked to skipped upload can be pruned (#1011)

changes/changelog.d/1086.bugfix

@@ -0,0 +1 @@
+Added safeguard to ensure local uploads are never purged from cache (#1086)

changes/changelog.d/1087.bugfix

@@ -0,0 +1 @@
+Fix playlist modal only listing 50 first playlists (#1087)

deploy/env.prod.sample

@@ -38,7 +38,8 @@ FUNKWHALE_API_PORT=5000
 # more concurrent requests, but also leads to higher CPU/Memory usage
 FUNKWHALE_WEB_WORKERS=1
 # Replace this by the definitive, public domain you will use for
-# your instance
+# your instance. It cannot be changed after initial deployment
+# without breaking your instance.
 FUNKWHALE_HOSTNAME=yourdomain.funkwhale
 FUNKWHALE_PROTOCOL=https
 

docs/installation/index.rst

@@ -1,6 +1,19 @@
 Installation
 =============
 
+Requirements
+------------
+
+Regardless of your chosen installation method, the following requirements must be met in order to successfully deploy Funkwhale:
+
+- **A dedicated domain or subdomain**: it is not possible to deploy Funkwhale on a subdirectory of an existing domain.
+- **Access to ports 80 and/or 443**: if you cannot serve the Funkwhale web app and API on these ports, federation will not work
+
+.. note::
+
+    Because of the federated nature of Funkwhale, **we strongly recommend you not to change the Funkwhale domain after initial deployment**, as it is likely to break
+    your installation.
+
 Available installation methods
 -------------------------------
 

front/src/store/playlists.js

@@ -25,14 +25,20 @@ export default {
     }
   },
   actions: {
-    fetchOwn ({commit, rootState}) {
+    async fetchOwn ({commit, rootState}) {
       let userId = rootState.auth.profile.id
       if (!userId) {
         return
       }
-      return axios.get('playlists/', {params: {user: userId}}).then((response) => {
-        commit('playlists', response.data.results)
-      })
+      let playlists = []
+      let url = 'playlists/'
+      while (url != null) {
+        let response = await axios.get(url, {params: {user: userId}})
+        playlists = [...playlists, ...response.data.results]
+        url = response.data.next
+
+      }
+      commit('playlists', playlists)
     }
   }
 }