From 1654044a9f48c3564f7ba03056f60c6e2c56b368 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 19 Mar 2020 14:41:15 +0100 Subject: [PATCH] See #170: store and compute modification date on artists --- api/funkwhale_api/audio/filters.py | 9 ++++- api/funkwhale_api/audio/serializers.py | 7 +++- api/funkwhale_api/common/serializers.py | 1 + api/funkwhale_api/common/utils.py | 17 ++++++++++ api/funkwhale_api/music/admin.py | 2 +- .../migrations/0051_auto_20200319_1249.py | 33 +++++++++++++++++++ api/funkwhale_api/music/models.py | 2 +- api/funkwhale_api/music/serializers.py | 1 + api/funkwhale_api/music/tasks.py | 2 ++ api/funkwhale_api/music/views.py | 10 ++++-- api/tests/audio/test_serializers.py | 3 +- api/tests/music/test_tasks.py | 5 +++ 12 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 api/funkwhale_api/music/migrations/0051_auto_20200319_1249.py diff --git a/api/funkwhale_api/audio/filters.py b/api/funkwhale_api/audio/filters.py index 9a8bec6cd..3a8da5ca7 100644 --- a/api/funkwhale_api/audio/filters.py +++ b/api/funkwhale_api/audio/filters.py @@ -28,10 +28,17 @@ class ChannelFilter(moderation_filters.HiddenContentFilterSet): subscribed = django_filters.BooleanFilter( field_name="_", method="filter_subscribed" ) + ordering = django_filters.OrderingFilter( + # tuple-mapping retains order + fields=( + ("creation_date", "creation_date"), + ("artist__modification_date", "modification_date"), + ) + ) class Meta: model = models.Channel - fields = ["q", "scope", "tag", "subscribed"] + fields = ["q", "scope", "tag", "subscribed", "ordering"] hidden_content_fields_mapping = moderation_filters.USER_FILTER_CONFIG["CHANNEL"] def filter_subscribed(self, queryset, name, value): diff --git a/api/funkwhale_api/audio/serializers.py b/api/funkwhale_api/audio/serializers.py index e99ad868e..cffa49b28 100644 --- a/api/funkwhale_api/audio/serializers.py +++ b/api/funkwhale_api/audio/serializers.py @@ -384,7 +384,12 @@ def get_channel_from_rss_url(url, raise_exception=False): library=channel.library, delete_existing=True, ) - + latest_upload_date = max([upload.creation_date for upload in uploads]) + if ( + not channel.artist.modification_date + or channel.artist.modification_date < latest_upload_date + ): + common_utils.update_modification_date(channel.artist) return channel, uploads diff --git a/api/funkwhale_api/common/serializers.py b/api/funkwhale_api/common/serializers.py index f1f332d13..f5565f133 100644 --- a/api/funkwhale_api/common/serializers.py +++ b/api/funkwhale_api/common/serializers.py @@ -77,6 +77,7 @@ class RelatedField(serializers.RelatedField): self.display_value(item), ) for item in queryset + if self.serializer ] ) diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index 7d750ff22..98fd21355 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -1,3 +1,5 @@ +import datetime + from django.core.files.base import ContentFile from django.utils.deconstruct import deconstructible @@ -14,6 +16,7 @@ from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit from django.conf import settings from django import urls from django.db import models, transaction +from django.utils import timezone logger = logging.getLogger(__name__) @@ -405,3 +408,17 @@ def get_mimetype_from_ext(path): def get_audio_mimetype(mt): aliases = {"audio/x-mp3": "audio/mpeg", "audio/mpeg3": "audio/mpeg"} return aliases.get(mt, mt) + + +def update_modification_date(obj, field="modification_date"): + IGNORE_DELAY = 60 + current_value = getattr(obj, field) + now = timezone.now() + ignore = current_value is not None and current_value < now - datetime.timedelta( + seconds=IGNORE_DELAY + ) + if ignore: + setattr(obj, field, now) + obj.__class__.objects.filter(pk=obj.pk).update(**{field: now}) + + return now diff --git a/api/funkwhale_api/music/admin.py b/api/funkwhale_api/music/admin.py index 56712746d..7e6a79450 100644 --- a/api/funkwhale_api/music/admin.py +++ b/api/funkwhale_api/music/admin.py @@ -5,7 +5,7 @@ from . import models @admin.register(models.Artist) class ArtistAdmin(admin.ModelAdmin): - list_display = ["name", "mbid", "creation_date"] + list_display = ["name", "mbid", "creation_date", "modification_date"] search_fields = ["name", "mbid"] diff --git a/api/funkwhale_api/music/migrations/0051_auto_20200319_1249.py b/api/funkwhale_api/music/migrations/0051_auto_20200319_1249.py new file mode 100644 index 000000000..e69e2035a --- /dev/null +++ b/api/funkwhale_api/music/migrations/0051_auto_20200319_1249.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.4 on 2020-03-19 12:49 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('music', '0050_auto_20200129_1344'), + ] + + operations = [ + migrations.RemoveField( + model_name='album', + name='cover', + ), + migrations.AddField( + model_name='artist', + name='modification_date', + field=models.DateTimeField(db_index=True, default=django.utils.timezone.now), + ), + migrations.AlterField( + model_name='upload', + name='import_status', + field=models.CharField(choices=[('draft', 'Draft'), ('pending', 'Pending'), ('finished', 'Finished'), ('errored', 'Errored'), ('skipped', 'Skipped')], default='pending', max_length=25), + ), + migrations.AlterField( + model_name='uploadversion', + name='mimetype', + field=models.CharField(choices=[('audio/mpeg3', 'mp3'), ('audio/x-mp3', 'mp3'), ('audio/mpeg', 'mp3'), ('video/ogg', 'ogg'), ('audio/ogg', 'ogg'), ('audio/opus', 'opus'), ('audio/x-m4a', 'aac'), ('audio/x-m4a', 'm4a'), ('audio/x-flac', 'flac'), ('audio/flac', 'flac')], max_length=50), + ), + ] diff --git a/api/funkwhale_api/music/models.py b/api/funkwhale_api/music/models.py index c7ae71ca8..cdfb819b7 100644 --- a/api/funkwhale_api/music/models.py +++ b/api/funkwhale_api/music/models.py @@ -251,7 +251,7 @@ class Artist(APIModelMixin): choices=ARTIST_CONTENT_CATEGORY_CHOICES, null=True, ) - + modification_date = models.DateTimeField(default=timezone.now, db_index=True) api = musicbrainz.api.artists objects = ArtistQuerySet.as_manager() diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py index 3357f14a6..9a26361b3 100644 --- a/api/funkwhale_api/music/serializers.py +++ b/api/funkwhale_api/music/serializers.py @@ -160,6 +160,7 @@ def serialize_artist_simple(artist): "mbid": str(artist.mbid), "name": artist.name, "creation_date": DATETIME_FIELD.to_representation(artist.creation_date), + "modification_date": DATETIME_FIELD.to_representation(artist.modification_date), "is_local": artist.is_local, "content_category": artist.content_category, } diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py index 1d53c75fb..89ee1a88f 100644 --- a/api/funkwhale_api/music/tasks.py +++ b/api/funkwhale_api/music/tasks.py @@ -289,6 +289,8 @@ def process_upload(upload, update_denormalization=True): "bitrate", ] ) + if channel: + common_utils.update_modification_date(channel.artist) if update_denormalization: models.TrackActor.create_entries( diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py index df82090bc..3b5c33a44 100644 --- a/api/funkwhale_api/music/views.py +++ b/api/funkwhale_api/music/views.py @@ -129,7 +129,7 @@ class ArtistViewSet( required_scope = "libraries" anonymous_policy = "setting" filterset_class = filters.ArtistFilter - ordering_fields = ("id", "name", "creation_date") + ordering_fields = ("id", "name", "creation_date", "modification_date") fetches = federation_decorators.fetches_route() mutations = common_decorators.mutations_route(types=["update"]) @@ -186,7 +186,12 @@ class AlbumViewSet( permission_classes = [oauth_permissions.ScopePermission] required_scope = "libraries" anonymous_policy = "setting" - ordering_fields = ("creation_date", "release_date", "title") + ordering_fields = ( + "creation_date", + "release_date", + "title", + "artist__modification_date", + ) filterset_class = filters.AlbumFilter fetches = federation_decorators.fetches_route() @@ -335,6 +340,7 @@ class TrackViewSet( "position", "disc_number", "artist__name", + "artist__modification_date", ) fetches = federation_decorators.fetches_route() mutations = common_decorators.mutations_route(types=["update"]) diff --git a/api/tests/audio/test_serializers.py b/api/tests/audio/test_serializers.py index f5672d982..f1dcd21c6 100644 --- a/api/tests/audio/test_serializers.py +++ b/api/tests/audio/test_serializers.py @@ -834,9 +834,9 @@ def test_get_channel_from_rss_url(db, r_mock, mocker): """ parsed_feed = feedparser.parse(xml_payload) - r_mock.get(rss_url, text=xml_payload) + update_modification_date = mocker.spy(common_utils, "update_modification_date") feed_init = mocker.spy(serializers.RssFeedSerializer, "__init__") feed_save = mocker.spy(serializers.RssFeedSerializer, "save") item_init = mocker.spy(serializers.RssFeedItemSerializer, "__init__") @@ -865,6 +865,7 @@ def test_get_channel_from_rss_url(db, r_mock, mocker): library=channel.library, delete_existing=True, ) + update_modification_date.assert_called_once_with(channel.artist) def test_get_channel_from_rss_honor_mrf_inbox_before_http( diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py index 35ed27694..b9bb018b8 100644 --- a/api/tests/music/test_tasks.py +++ b/api/tests/music/test_tasks.py @@ -6,6 +6,7 @@ import uuid from django.core.paginator import Paginator from django.utils import timezone +from funkwhale_api.common import utils as common_utils from funkwhale_api.federation import serializers as federation_serializers from funkwhale_api.federation import jsonld from funkwhale_api.federation import utils as federation_utils @@ -1040,6 +1041,8 @@ def test_process_channel_upload_forces_artist_and_attributed_to( factories, mocker, faker ): channel = factories["audio.Channel"](attributed_to__local=True) + update_modification_date = mocker.spy(common_utils, "update_modification_date") + attachment = factories["common.Attachment"](actor=channel.attributed_to) import_metadata = { "title": "Real title", @@ -1081,6 +1084,8 @@ def test_process_channel_upload_forces_artist_and_attributed_to( assert upload.track.attributed_to == channel.attributed_to assert upload.track.attachment_cover == attachment + update_modification_date.assert_called_once_with(channel.artist) + def test_process_upload_uses_import_metadata_if_valid(factories, mocker): track = factories["music.Track"]()