From 9333fdc62bd79e6fd24f2d5bc48136127d8c676f Mon Sep 17 00:00:00 2001 From: Agate Date: Thu, 7 May 2020 13:14:12 +0200 Subject: [PATCH 1/4] Fix #1087: Fix playlist modal only listing 50 first playlists --- changes/changelog.d/1087.bugfix | 1 + front/src/store/playlists.js | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 changes/changelog.d/1087.bugfix diff --git a/changes/changelog.d/1087.bugfix b/changes/changelog.d/1087.bugfix new file mode 100644 index 000000000..36293d96a --- /dev/null +++ b/changes/changelog.d/1087.bugfix @@ -0,0 +1 @@ +Fix playlist modal only listing 50 first playlists (#1087) diff --git a/front/src/store/playlists.js b/front/src/store/playlists.js index d0e144d80..60f8771ac 100644 --- a/front/src/store/playlists.js +++ b/front/src/store/playlists.js @@ -25,14 +25,20 @@ export default { } }, actions: { - fetchOwn ({commit, rootState}) { + async fetchOwn ({commit, rootState}) { let userId = rootState.auth.profile.id if (!userId) { return } - return axios.get('playlists/', {params: {user: userId}}).then((response) => { - commit('playlists', response.data.results) - }) + let playlists = [] + let url = 'playlists/' + while (url != null) { + let response = await axios.get(url, {params: {user: userId}}) + playlists = [...playlists, ...response.data.results] + url = response.data.next + + } + commit('playlists', playlists) } } } From c7a3dd9da59b0b3ad956f4f00c9b5b291626c6a6 Mon Sep 17 00:00:00 2001 From: Agate Date: Thu, 7 May 2020 13:51:29 +0200 Subject: [PATCH 2/4] Fix #1011: Ensure tracks linked to skipped upload can be pruned --- api/funkwhale_api/music/tasks.py | 10 ++++++++-- api/tests/music/test_tasks.py | 2 ++ changes/changelog.d/1011.bugfix | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changes/changelog.d/1011.bugfix diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index 722a9eefb..de2d00315 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -817,9 +817,15 @@ def get_prunable_tracks( Returns a list of tracks with no associated uploads, excluding the one that were listened/favorited/included in playlists. """ - + purgeable_tracks_with_upload = ( + models.Upload.objects.exclude(track=None) + .filter(import_status="skipped") + .values("track") + ) queryset = models.Track.objects.all() - queryset = queryset.filter(uploads__isnull=True) + queryset = queryset.filter( + Q(uploads__isnull=True) | Q(pk__in=purgeable_tracks_with_upload) + ) if exclude_favorites: queryset = queryset.filter(track_favorites__isnull=True) if exclude_playlists: diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py index 96ba451aa..989382d79 100644 --- a/api/tests/music/test_tasks.py +++ b/api/tests/music/test_tasks.py @@ -867,6 +867,8 @@ def test_clean_transcoding_cache(preferences, now, factories): def test_get_prunable_tracks(factories): prunable_track = factories["music.Track"]() + # track is still prunable if it has a skipped upload linked to it + factories["music.Upload"](import_status="skipped", track=prunable_track) # non prunable tracks factories["music.Upload"]() factories["favorites.TrackFavorite"]() diff --git a/changes/changelog.d/1011.bugfix b/changes/changelog.d/1011.bugfix new file mode 100644 index 000000000..0b52c0e6e --- /dev/null +++ b/changes/changelog.d/1011.bugfix @@ -0,0 +1 @@ +Ensure tracks linked to skipped upload can be pruned (#1011) From 49a8b2babff345dc9fca7bd305b0aab99d316fe5 Mon Sep 17 00:00:00 2001 From: Agate Date: Thu, 7 May 2020 13:39:57 +0200 Subject: [PATCH 3/4] Fix #1086: Added safeguard to ensure local uploads are never purged from cache --- api/funkwhale_api/federation/tasks.py | 1 + api/tests/federation/test_tasks.py | 15 +++++++++++++-- changes/changelog.d/1086.bugfix | 1 + deploy/env.prod.sample | 3 ++- docs/installation/index.rst | 13 +++++++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 changes/changelog.d/1086.bugfix diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index f802c5111..d9b0e6083 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -50,6 +50,7 @@ def clean_music_cache(): ) .local(False) .exclude(audio_file="") + .filter(Q(source__startswith="http://") | Q(source__startswith="https://")) .only("audio_file", "id") .order_by("id") ) diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 79573ff45..6974cf11f 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -16,20 +16,28 @@ def test_clean_federation_music_cache_if_no_listen(preferences, factories): preferences["federation__music_cache_duration"] = 60 remote_library = factories["music.Library"]() upload1 = factories["music.Upload"]( - library=remote_library, accessed_date=timezone.now() + library=remote_library, + accessed_date=timezone.now(), + source="https://upload1.mp3", ) upload2 = factories["music.Upload"]( library=remote_library, accessed_date=timezone.now() - datetime.timedelta(minutes=61), + source="https://upload2.mp3", + ) + upload3 = factories["music.Upload"]( + library=remote_library, accessed_date=None, source="http://upload3.mp3" ) - upload3 = factories["music.Upload"](library=remote_library, accessed_date=None) # local upload, should not be cleaned upload4 = factories["music.Upload"](library__actor__local=True, accessed_date=None) + # non-http source , not cleaned + upload5 = factories["music.Upload"](accessed_date=None, source="noop") path1 = upload1.audio_file_path path2 = upload2.audio_file_path path3 = upload3.audio_file_path path4 = upload4.audio_file_path + path5 = upload5.audio_file_path tasks.clean_music_cache() @@ -37,15 +45,18 @@ def test_clean_federation_music_cache_if_no_listen(preferences, factories): upload2.refresh_from_db() upload3.refresh_from_db() upload4.refresh_from_db() + upload5.refresh_from_db() assert bool(upload1.audio_file) is True assert bool(upload2.audio_file) is False assert bool(upload3.audio_file) is False assert bool(upload4.audio_file) is True + assert bool(upload5.audio_file) is True assert os.path.exists(path1) is True assert os.path.exists(path2) is False assert os.path.exists(path3) is False assert os.path.exists(path4) is True + assert os.path.exists(path5) is True def test_clean_federation_music_cache_orphaned(settings, preferences, factories): diff --git a/changes/changelog.d/1086.bugfix b/changes/changelog.d/1086.bugfix new file mode 100644 index 000000000..db7c38dda --- /dev/null +++ b/changes/changelog.d/1086.bugfix @@ -0,0 +1 @@ +Added safeguard to ensure local uploads are never purged from cache (#1086) diff --git a/deploy/env.prod.sample b/deploy/env.prod.sample index 4a184e833..cb567e20d 100644 --- a/deploy/env.prod.sample +++ b/deploy/env.prod.sample @@ -38,7 +38,8 @@ FUNKWHALE_API_PORT=5000 # more concurrent requests, but also leads to higher CPU/Memory usage FUNKWHALE_WEB_WORKERS=1 # Replace this by the definitive, public domain you will use for -# your instance +# your instance. It cannot be changed after initial deployment +# without breaking your instance. FUNKWHALE_HOSTNAME=yourdomain.funkwhale FUNKWHALE_PROTOCOL=https diff --git a/docs/installation/index.rst b/docs/installation/index.rst index 4d05ba118..5f3a78c2e 100644 --- a/docs/installation/index.rst +++ b/docs/installation/index.rst @@ -1,6 +1,19 @@ Installation ============= +Requirements +------------ + +Regardless of your chosen installation method, the following requirements must be met in order to successfully deploy Funkwhale: + +- **A dedicated domain or subdomain**: it is not possible to deploy Funkwhale on a subdirectory of an existing domain. +- **Access to ports 80 and/or 443**: if you cannot serve the Funkwhale web app and API on these ports, federation will not work + +.. note:: + + Because of the federated nature of Funkwhale, **we strongly recommend you not to change the Funkwhale domain after initial deployment**, as it is likely to break + your installation. + Available installation methods ------------------------------- From 5415fc540e1d564b64d65d015716e7c6fd700c92 Mon Sep 17 00:00:00 2001 From: Agate Date: Thu, 7 May 2020 14:05:24 +0200 Subject: [PATCH 4/4] Build docker images for master branch --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a089db796..25d6ccb04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -224,7 +224,7 @@ docker_release: before_script: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - cp -r front/dist api/frontend - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); + - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); script: - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi - cd api @@ -233,6 +233,7 @@ docker_release: - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $IMAGE_LATEST; fi only: - develop@funkwhale/funkwhale + - master@funkwhale/funkwhale - tags@funkwhale/funkwhale tags: - docker-build @@ -246,7 +247,7 @@ docker_all_in_one_release: BUILD_PATH: all_in_one before_script: - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); + - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); script: - if [[ ! -z "$CI_COMMIT_TAG" ]]; then (./docs/get-releases-json.py | scripts/is-docker-latest.py $CI_COMMIT_TAG -) && export DOCKER_LATEST_TAG="-t $ALL_IN_ONE_IMAGE_LATEST" || export DOCKER_LATEST_TAG=; fi - wget $ALL_IN_ONE_ARTIFACT_URL -O all_in_one.zip @@ -261,6 +262,7 @@ docker_all_in_one_release: - if [[ ! -z "$DOCKER_LATEST_TAG" ]]; then docker push $ALL_IN_ONE_IMAGE_LATEST; fi only: - develop@funkwhale/funkwhale + - master@funkwhale/funkwhale - tags@funkwhale/funkwhale tags: - docker-build @@ -275,7 +277,7 @@ build_api: - api script: - rm -rf api/tests - - (if [ "$CI_COMMIT_REF_NAME" == "develop" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); + - (if [ "$CI_COMMIT_REF_NAME" == "develop" ] || [ "$CI_COMMIT_REF_NAME" == "master" ]; then ./scripts/set-api-build-metadata.sh $(echo $CI_COMMIT_SHA | cut -c 1-8); fi); - chmod -R 750 api - echo Done! only: