From 9c22a72ed18f636632f0d4fd98e2bd313fb76482 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 30 Jan 2020 17:28:52 +0100 Subject: [PATCH] See #170: RSS feeds for channels --- api/funkwhale_api/audio/categories.py | 111 +++++++++ api/funkwhale_api/audio/factories.py | 1 + .../audio/migrations/0002_channel_metadata.py | 21 ++ api/funkwhale_api/audio/models.py | 21 ++ api/funkwhale_api/audio/renderers.py | 32 +++ api/funkwhale_api/audio/serializers.py | 217 +++++++++++++++++- api/funkwhale_api/audio/spa_views.py | 10 + api/funkwhale_api/audio/views.py | 41 +++- api/funkwhale_api/common/locales.py | 191 +++++++++++++++ api/funkwhale_api/common/models.py | 14 ++ api/funkwhale_api/common/utils.py | 4 + api/funkwhale_api/federation/models.py | 4 + api/funkwhale_api/subsonic/renderers.py | 2 + api/tests/audio/test_serializers.py | 214 +++++++++++++++++ api/tests/audio/test_spa_views.py | 7 + api/tests/audio/test_views.py | 19 ++ api/tests/common/test_models.py | 16 ++ docker/traefik.toml | 4 +- 18 files changed, 923 insertions(+), 6 deletions(-) create mode 100644 api/funkwhale_api/audio/categories.py create mode 100644 api/funkwhale_api/audio/migrations/0002_channel_metadata.py create mode 100644 api/funkwhale_api/audio/renderers.py create mode 100644 api/funkwhale_api/common/locales.py diff --git a/api/funkwhale_api/audio/categories.py b/api/funkwhale_api/audio/categories.py new file mode 100644 index 000000000..56a748a53 --- /dev/null +++ b/api/funkwhale_api/audio/categories.py @@ -0,0 +1,111 @@ +# from https://help.apple.com/itc/podcasts_connect/#/itc9267a2f12 +ITUNES_CATEGORIES = { + "Arts": [ + "Books", + "Design", + "Fashion & Beauty", + "Food", + "Performing Arts", + "Visual Arts", + ], + "Business": [ + "Careers", + "Entrepreneurship", + "Investing", + "Management", + "Marketing", + "Non-Profit", + ], + "Comedy": ["Comedy Interviews", "Improv", "Stand-Up"], + "Education": ["Courses", "How To", "Language Learning", "Self-Improvement"], + "Fiction": ["Comedy Fiction", "Drama", "Science Fiction"], + "Government": [], + "History": [], + "Health & Fitness": [ + "Alternative Health", + "Fitness", + "Medicine", + "Mental Health", + "Nutrition", + "Sexuality", + ], + "Kids & Family": [ + "Education for Kids", + "Parenting", + "Pets & Animals", + "Stories for Kids", + ], + "Leisure": [ + "Animation & Manga", + "Automotive", + "Aviation", + "Crafts", + "Games", + "Hobbies", + "Home & Garden", + "Video Games", + ], + "Music": ["Music Commentary", "Music History", "Music Interviews"], + "News": [ + "Business News", + "Daily News", + "Entertainment News", + "News Commentary", + "Politics", + "Sports News", + "Tech News", + ], + "Religion & Spirituality": [ + "Buddhism", + "Christianity", + "Hinduism", + "Islam", + "Judaism", + "Religion", + "Spirituality", + ], + "Science": [ + "Astronomy", + "Chemistry", + "Earth Sciences", + "Life Sciences", + "Mathematics", + "Natural Sciences", + "Nature", + "Physics", + "Social Sciences", + ], + "Society & Culture": [ + "Documentary", + "Personal Journals", + "Philosophy", + "Places & Travel", + "Relationships", + ], + "Sports": [ + "Baseball", + "Basketball", + "Cricket", + "Fantasy Sports", + "Football", + "Golf", + "Hockey", + "Rugby", + "Running", + "Soccer", + "Swimming", + "Tennis", + "Volleyball", + "Wilderness", + "Wrestling", + ], + "Technology": [], + "True Crime": [], + "TV & Film": [ + "After Shows", + "Film History", + "Film Interviews", + "Film Reviews", + "TV Reviews", + ], +} diff --git a/api/funkwhale_api/audio/factories.py b/api/funkwhale_api/audio/factories.py index 3704b4d1d..9629b2a1e 100644 --- a/api/funkwhale_api/audio/factories.py +++ b/api/funkwhale_api/audio/factories.py @@ -25,6 +25,7 @@ class ChannelFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory): music_factories.ArtistFactory, attributed_to=factory.SelfAttribute("..attributed_to"), ) + metadata = factory.LazyAttribute(lambda o: {}) class Meta: model = "audio.Channel" diff --git a/api/funkwhale_api/audio/migrations/0002_channel_metadata.py b/api/funkwhale_api/audio/migrations/0002_channel_metadata.py new file mode 100644 index 000000000..c7a1806f1 --- /dev/null +++ b/api/funkwhale_api/audio/migrations/0002_channel_metadata.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.9 on 2020-01-31 06:24 + +import django.contrib.postgres.fields.jsonb +import django.core.serializers.json +from django.db import migrations +import funkwhale_api.audio.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('audio', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='channel', + name='metadata', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=funkwhale_api.audio.models.empty_dict, encoder=django.core.serializers.json.DjangoJSONEncoder, max_length=50000), + ), + ] diff --git a/api/funkwhale_api/audio/models.py b/api/funkwhale_api/audio/models.py index f3f9db896..7acca926a 100644 --- a/api/funkwhale_api/audio/models.py +++ b/api/funkwhale_api/audio/models.py @@ -1,14 +1,22 @@ import uuid +from django.contrib.postgres.fields import JSONField +from django.core.serializers.json import DjangoJSONEncoder from django.db import models +from django.urls import reverse from django.utils import timezone from funkwhale_api.federation import keys from funkwhale_api.federation import models as federation_models +from funkwhale_api.federation import utils as federation_utils from funkwhale_api.users import models as user_models +def empty_dict(): + return {} + + class Channel(models.Model): uuid = models.UUIDField(default=uuid.uuid4, unique=True) artist = models.OneToOneField( @@ -29,6 +37,19 @@ class Channel(models.Model): ) creation_date = models.DateTimeField(default=timezone.now) + # metadata to enhance rss feed + metadata = JSONField( + default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True + ) + + def get_absolute_url(self): + return federation_utils.full_url("/channels/{}".format(self.uuid)) + + def get_rss_url(self): + return federation_utils.full_url( + reverse("api:v1:channels-rss", kwargs={"uuid": self.uuid}) + ) + def generate_actor(username, **kwargs): actor_data = user_models.get_actor_data(username, **kwargs) diff --git a/api/funkwhale_api/audio/renderers.py b/api/funkwhale_api/audio/renderers.py new file mode 100644 index 000000000..0a8e71d6b --- /dev/null +++ b/api/funkwhale_api/audio/renderers.py @@ -0,0 +1,32 @@ +import xml.etree.ElementTree as ET + +from rest_framework import negotiation +from rest_framework import renderers + +from funkwhale_api.subsonic.renderers import dict_to_xml_tree + + +class PodcastRSSRenderer(renderers.JSONRenderer): + media_type = "application/rss+xml" + + def render(self, data, accepted_media_type=None, renderer_context=None): + if not data: + # when stream view is called, we don't have any data + return super().render(data, accepted_media_type, renderer_context) + final = { + "version": "2.0", + "xmlns:atom": "http://www.w3.org/2005/Atom", + "xmlns:itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd", + "xmlns:media": "http://search.yahoo.com/mrss/", + } + final.update(data) + tree = dict_to_xml_tree("rss", final) + return b'\n' + ET.tostring( + tree, encoding="utf-8" + ) + + +class PodcastRSSContentNegociation(negotiation.DefaultContentNegotiation): + def select_renderer(self, request, renderers, format_suffix=None): + + return (PodcastRSSRenderer(), PodcastRSSRenderer.media_type) diff --git a/api/funkwhale_api/audio/serializers.py b/api/funkwhale_api/audio/serializers.py index 3971a070e..0c9732efe 100644 --- a/api/funkwhale_api/audio/serializers.py +++ b/api/funkwhale_api/audio/serializers.py @@ -4,15 +4,50 @@ from rest_framework import serializers from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import utils as common_utils +from funkwhale_api.common import locales from funkwhale_api.federation import serializers as federation_serializers +from funkwhale_api.federation import utils as federation_utils from funkwhale_api.music import models as music_models from funkwhale_api.music import serializers as music_serializers from funkwhale_api.tags import models as tags_models from funkwhale_api.tags import serializers as tags_serializers +from . import categories from . import models +class ChannelMetadataSerializer(serializers.Serializer): + itunes_category = serializers.ChoiceField( + choices=categories.ITUNES_CATEGORIES, required=True + ) + itunes_subcategory = serializers.CharField(required=False) + language = serializers.ChoiceField(required=True, choices=locales.ISO_639_CHOICES) + copyright = serializers.CharField(required=False, allow_null=True, max_length=255) + owner_name = serializers.CharField(required=False, allow_null=True, max_length=255) + owner_email = serializers.EmailField(required=False, allow_null=True) + explicit = serializers.BooleanField(required=False) + + def validate(self, validated_data): + validated_data = super().validate(validated_data) + subcategory = self._validate_itunes_subcategory( + validated_data["itunes_category"], validated_data.get("itunes_subcategory") + ) + if subcategory: + validated_data["itunes_subcategory"] = subcategory + return validated_data + + def _validate_itunes_subcategory(self, parent, child): + if not child: + return + + if child not in categories.ITUNES_CATEGORIES[parent]: + raise serializers.ValidationError( + '"{}" is not a valid subcategory for "{}"'.format(child, parent) + ) + + return child + + class ChannelCreateSerializer(serializers.Serializer): name = serializers.CharField(max_length=music_models.MAX_LENGTHS["ARTIST_NAME"]) username = serializers.CharField(max_length=music_models.MAX_LENGTHS["ARTIST_NAME"]) @@ -21,6 +56,17 @@ class ChannelCreateSerializer(serializers.Serializer): content_category = serializers.ChoiceField( choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES ) + metadata = serializers.DictField(required=False) + + def validate(self, validated_data): + validated_data = super().validate(validated_data) + metadata = validated_data.pop("metadata", {}) + if validated_data["content_category"] == "podcast": + metadata_serializer = ChannelMetadataSerializer(data=metadata) + metadata_serializer.is_valid(raise_exception=True) + metadata = metadata_serializer.validated_data + validated_data["metadata"] = metadata + return validated_data @transaction.atomic def create(self, validated_data): @@ -38,7 +84,9 @@ class ChannelCreateSerializer(serializers.Serializer): tags_models.set_tags(artist, *validated_data["tags"]) channel = models.Channel( - artist=artist, attributed_to=validated_data["attributed_to"] + artist=artist, + attributed_to=validated_data["attributed_to"], + metadata=validated_data["metadata"], ) summary = description_obj.rendered if description_obj else None channel.actor = models.generate_actor( @@ -57,6 +105,9 @@ class ChannelCreateSerializer(serializers.Serializer): return ChannelSerializer(obj).data +NOOP = object() + + class ChannelUpdateSerializer(serializers.Serializer): name = serializers.CharField(max_length=music_models.MAX_LENGTHS["ARTIST_NAME"]) description = common_serializers.ContentSerializer(allow_null=True) @@ -64,6 +115,32 @@ class ChannelUpdateSerializer(serializers.Serializer): content_category = serializers.ChoiceField( choices=music_models.ARTIST_CONTENT_CATEGORY_CHOICES ) + metadata = serializers.DictField(required=False) + + def validate(self, validated_data): + validated_data = super().validate(validated_data) + require_metadata_validation = False + new_content_category = validated_data.get("content_category") + metadata = validated_data.pop("metadata", NOOP) + if ( + new_content_category == "podcast" + and self.instance.artist.content_category != "postcast" + ): + # updating channel, setting as podcast + require_metadata_validation = True + elif self.instance.artist.content_category == "postcast" and metadata != NOOP: + # channel is podcast, and metadata was updated + require_metadata_validation = True + else: + metadata = self.instance.metadata + + if require_metadata_validation: + metadata_serializer = ChannelMetadataSerializer(data=metadata) + metadata_serializer.is_valid(raise_exception=True) + metadata = metadata_serializer.validated_data + + validated_data["metadata"] = metadata + return validated_data @transaction.atomic def update(self, obj, validated_data): @@ -72,6 +149,9 @@ class ChannelUpdateSerializer(serializers.Serializer): actor_update_fields = [] artist_update_fields = [] + obj.metadata = validated_data["metadata"] + obj.save(update_fields=["metadata"]) + if "description" in validated_data: description_obj = common_utils.attach_content( obj.artist, "description", validated_data["description"] @@ -111,7 +191,14 @@ class ChannelSerializer(serializers.ModelSerializer): class Meta: model = models.Channel - fields = ["uuid", "artist", "attributed_to", "actor", "creation_date"] + fields = [ + "uuid", + "artist", + "attributed_to", + "actor", + "creation_date", + "metadata", + ] def get_artist(self, obj): return music_serializers.serialize_artist_simple(obj.artist) @@ -136,3 +223,129 @@ class SubscriptionSerializer(serializers.Serializer): data = super().to_representation(obj) data["channel"] = ChannelSerializer(obj.target.channel).data return data + + +# RSS related stuff +# https://github.com/simplepie/simplepie-ng/wiki/Spec:-iTunes-Podcast-RSS +# is extremely useful + + +def rss_date(dt): + return dt.strftime("%a, %d %b %Y %H:%M:%S %z") + + +def rss_duration(seconds): + if not seconds: + return "00:00:00" + full_hours = seconds // 3600 + full_minutes = (seconds - (full_hours * 3600)) // 60 + remaining_seconds = seconds - (full_hours * 3600) - (full_minutes * 60) + return "{}:{}:{}".format( + str(full_hours).zfill(2), + str(full_minutes).zfill(2), + str(remaining_seconds).zfill(2), + ) + + +def rss_serialize_item(upload): + data = { + "title": [{"value": upload.track.title}], + "itunes:title": [{"value": upload.track.title}], + "guid": [{"cdata_value": str(upload.uuid), "isPermalink": "false"}], + "pubDate": [{"value": rss_date(upload.creation_date)}], + "itunes:duration": [{"value": rss_duration(upload.duration)}], + "itunes:explicit": [{"value": "no"}], + "itunes:episodeType": [{"value": "full"}], + "itunes:season": [{"value": upload.track.disc_number or 1}], + "itunes:episode": [{"value": upload.track.position or 1}], + "link": [{"value": federation_utils.full_url(upload.track.get_absolute_url())}], + "enclosure": [ + { + "url": upload.listen_url, + "length": upload.size or 0, + "type": upload.mimetype or "audio/mpeg", + } + ], + } + if upload.track.description: + data["itunes:subtitle"] = [{"value": upload.track.description.truncate(255)}] + data["itunes:summary"] = [{"cdata_value": upload.track.description.rendered}] + data["description"] = [{"value": upload.track.description.as_plain_text}] + data["content:encoded"] = data["itunes:summary"] + + if upload.track.attachment_cover: + data["itunes:image"] = [ + {"href": upload.track.attachment_cover.download_url_original} + ] + + tagged_items = getattr(upload.track, "_prefetched_tagged_items", []) + if tagged_items: + data["itunes:keywords"] = [ + {"value": " ".join([ti.tag.name for ti in tagged_items])} + ] + + return data + + +def rss_serialize_channel(channel): + metadata = channel.metadata or {} + explicit = metadata.get("explicit", False) + copyright = metadata.get("copyright", "All rights reserved") + owner_name = metadata.get("owner_name", channel.attributed_to.display_name) + owner_email = metadata.get("owner_email") + itunes_category = metadata.get("itunes_category") + itunes_subcategory = metadata.get("itunes_subcategory") + language = metadata.get("language") + + data = { + "title": [{"value": channel.artist.name}], + "copyright": [{"value": copyright}], + "itunes:explicit": [{"value": "no" if not explicit else "yes"}], + "itunes:author": [{"value": owner_name}], + "itunes:owner": [{"itunes:name": [{"value": owner_name}]}], + "itunes:type": [{"value": "episodic"}], + "link": [{"value": channel.get_absolute_url()}], + "atom:link": [ + { + "href": channel.get_rss_url(), + "rel": "self", + "type": "application/rss+xml", + } + ], + } + if language: + data["language"] = [{"value": language}] + + if owner_email: + data["itunes:owner"][0]["itunes:email"] = [{"value": owner_email}] + + if itunes_category: + node = {"text": itunes_category} + if itunes_subcategory: + node["itunes:category"] = [{"text": itunes_subcategory}] + data["itunes:category"] = [node] + + if channel.artist.description: + data["itunes:subtitle"] = [{"value": channel.artist.description.truncate(255)}] + data["itunes:summary"] = [{"cdata_value": channel.artist.description.rendered}] + data["description"] = [{"value": channel.artist.description.as_plain_text}] + + if channel.artist.attachment_cover: + data["itunes:image"] = [ + {"href": channel.artist.attachment_cover.download_url_original} + ] + + tagged_items = getattr(channel.artist, "_prefetched_tagged_items", []) + + if tagged_items: + data["itunes:keywords"] = [ + {"value": " ".join([ti.tag.name for ti in tagged_items])} + ] + + return data + + +def rss_serialize_channel_full(channel, uploads): + channel_data = rss_serialize_channel(channel) + channel_data["item"] = [rss_serialize_item(upload) for upload in uploads] + return {"channel": channel_data} diff --git a/api/funkwhale_api/audio/spa_views.py b/api/funkwhale_api/audio/spa_views.py index 34404812d..097e00cf4 100644 --- a/api/funkwhale_api/audio/spa_views.py +++ b/api/funkwhale_api/audio/spa_views.py @@ -47,6 +47,16 @@ def channel_detail(request, uuid): } ) + metas.append( + { + "tag": "link", + "rel": "alternate", + "type": "application/rss+xml", + "href": obj.get_rss_url(), + "title": "{} - RSS Podcast Feed".format(obj.artist.name), + }, + ) + if obj.library.uploads.all().playable_by(None).exists(): metas.append( { diff --git a/api/funkwhale_api/audio/views.py b/api/funkwhale_api/audio/views.py index 5162730a6..1f40dd0a6 100644 --- a/api/funkwhale_api/audio/views.py +++ b/api/funkwhale_api/audio/views.py @@ -6,14 +6,17 @@ from rest_framework import response from rest_framework import viewsets from django import http +from django.db.models import Prefetch from django.db.utils import IntegrityError from funkwhale_api.common import permissions from funkwhale_api.common import preferences from funkwhale_api.federation import models as federation_models +from funkwhale_api.music import models as music_models +from funkwhale_api.music import views as music_views from funkwhale_api.users.oauth import permissions as oauth_permissions -from . import filters, models, serializers +from . import filters, models, renderers, serializers class ChannelsMixin(object): @@ -37,7 +40,17 @@ class ChannelViewSet( serializer_class = serializers.ChannelSerializer queryset = ( models.Channel.objects.all() - .prefetch_related("library", "attributed_to", "artist__description", "actor") + .prefetch_related( + "library", + "attributed_to", + "actor", + Prefetch( + "artist", + queryset=music_models.Artist.objects.select_related( + "attachment_cover", "description" + ).prefetch_related(music_views.TAG_PREFETCH,), + ), + ) .order_by("-creation_date") ) permission_classes = [ @@ -92,6 +105,30 @@ class ChannelViewSet( request.user.actor.emitted_follows.filter(target=object.actor).delete() return response.Response(status=204) + @decorators.action( + detail=True, + methods=["get"], + permission_classes=[], + content_negotiation_class=renderers.PodcastRSSContentNegociation, + ) + def rss(self, request, *args, **kwargs): + object = self.get_object() + uploads = ( + object.library.uploads.playable_by(None) + .prefetch_related( + Prefetch( + "track", + queryset=music_models.Track.objects.select_related( + "attachment_cover", "description" + ).prefetch_related(music_views.TAG_PREFETCH,), + ), + ) + .select_related("track__attachment_cover", "track__description") + .order_by("-creation_date") + )[:50] + data = serializers.rss_serialize_channel_full(channel=object, uploads=uploads) + return response.Response(data, status=200) + def get_serializer_context(self): context = super().get_serializer_context() context["subscriptions_count"] = self.action in ["retrieve", "create", "update"] diff --git a/api/funkwhale_api/common/locales.py b/api/funkwhale_api/common/locales.py new file mode 100644 index 000000000..4c59276df --- /dev/null +++ b/api/funkwhale_api/common/locales.py @@ -0,0 +1,191 @@ +# from https://gist.github.com/carlopires/1262033/c52ef0f7ce4f58108619508308372edd8d0bd518 + +ISO_639_CHOICES = [ + ("ab", "Abkhaz"), + ("aa", "Afar"), + ("af", "Afrikaans"), + ("ak", "Akan"), + ("sq", "Albanian"), + ("am", "Amharic"), + ("ar", "Arabic"), + ("an", "Aragonese"), + ("hy", "Armenian"), + ("as", "Assamese"), + ("av", "Avaric"), + ("ae", "Avestan"), + ("ay", "Aymara"), + ("az", "Azerbaijani"), + ("bm", "Bambara"), + ("ba", "Bashkir"), + ("eu", "Basque"), + ("be", "Belarusian"), + ("bn", "Bengali"), + ("bh", "Bihari"), + ("bi", "Bislama"), + ("bs", "Bosnian"), + ("br", "Breton"), + ("bg", "Bulgarian"), + ("my", "Burmese"), + ("ca", "Catalan; Valencian"), + ("ch", "Chamorro"), + ("ce", "Chechen"), + ("ny", "Chichewa; Chewa; Nyanja"), + ("zh", "Chinese"), + ("cv", "Chuvash"), + ("kw", "Cornish"), + ("co", "Corsican"), + ("cr", "Cree"), + ("hr", "Croatian"), + ("cs", "Czech"), + ("da", "Danish"), + ("dv", "Divehi; Maldivian;"), + ("nl", "Dutch"), + ("dz", "Dzongkha"), + ("en", "English"), + ("eo", "Esperanto"), + ("et", "Estonian"), + ("ee", "Ewe"), + ("fo", "Faroese"), + ("fj", "Fijian"), + ("fi", "Finnish"), + ("fr", "French"), + ("ff", "Fula"), + ("gl", "Galician"), + ("ka", "Georgian"), + ("de", "German"), + ("el", "Greek, Modern"), + ("gn", "Guaraní"), + ("gu", "Gujarati"), + ("ht", "Haitian"), + ("ha", "Hausa"), + ("he", "Hebrew (modern)"), + ("hz", "Herero"), + ("hi", "Hindi"), + ("ho", "Hiri Motu"), + ("hu", "Hungarian"), + ("ia", "Interlingua"), + ("id", "Indonesian"), + ("ie", "Interlingue"), + ("ga", "Irish"), + ("ig", "Igbo"), + ("ik", "Inupiaq"), + ("io", "Ido"), + ("is", "Icelandic"), + ("it", "Italian"), + ("iu", "Inuktitut"), + ("ja", "Japanese"), + ("jv", "Javanese"), + ("kl", "Kalaallisut"), + ("kn", "Kannada"), + ("kr", "Kanuri"), + ("ks", "Kashmiri"), + ("kk", "Kazakh"), + ("km", "Khmer"), + ("ki", "Kikuyu, Gikuyu"), + ("rw", "Kinyarwanda"), + ("ky", "Kirghiz, Kyrgyz"), + ("kv", "Komi"), + ("kg", "Kongo"), + ("ko", "Korean"), + ("ku", "Kurdish"), + ("kj", "Kwanyama, Kuanyama"), + ("la", "Latin"), + ("lb", "Luxembourgish"), + ("lg", "Luganda"), + ("li", "Limburgish"), + ("ln", "Lingala"), + ("lo", "Lao"), + ("lt", "Lithuanian"), + ("lu", "Luba-Katanga"), + ("lv", "Latvian"), + ("gv", "Manx"), + ("mk", "Macedonian"), + ("mg", "Malagasy"), + ("ms", "Malay"), + ("ml", "Malayalam"), + ("mt", "Maltese"), + ("mi", "Māori"), + ("mr", "Marathi (Marāṭhī)"), + ("mh", "Marshallese"), + ("mn", "Mongolian"), + ("na", "Nauru"), + ("nv", "Navajo, Navaho"), + ("nb", "Norwegian Bokmål"), + ("nd", "North Ndebele"), + ("ne", "Nepali"), + ("ng", "Ndonga"), + ("nn", "Norwegian Nynorsk"), + ("no", "Norwegian"), + ("ii", "Nuosu"), + ("nr", "South Ndebele"), + ("oc", "Occitan"), + ("oj", "Ojibwe, Ojibwa"), + ("cu", "Old Church Slavonic"), + ("om", "Oromo"), + ("or", "Oriya"), + ("os", "Ossetian, Ossetic"), + ("pa", "Panjabi, Punjabi"), + ("pi", "Pāli"), + ("fa", "Persian"), + ("pl", "Polish"), + ("ps", "Pashto, Pushto"), + ("pt", "Portuguese"), + ("qu", "Quechua"), + ("rm", "Romansh"), + ("rn", "Kirundi"), + ("ro", "Romanian, Moldavan"), + ("ru", "Russian"), + ("sa", "Sanskrit (Saṁskṛta)"), + ("sc", "Sardinian"), + ("sd", "Sindhi"), + ("se", "Northern Sami"), + ("sm", "Samoan"), + ("sg", "Sango"), + ("sr", "Serbian"), + ("gd", "Scottish Gaelic"), + ("sn", "Shona"), + ("si", "Sinhala, Sinhalese"), + ("sk", "Slovak"), + ("sl", "Slovene"), + ("so", "Somali"), + ("st", "Southern Sotho"), + ("es", "Spanish; Castilian"), + ("su", "Sundanese"), + ("sw", "Swahili"), + ("ss", "Swati"), + ("sv", "Swedish"), + ("ta", "Tamil"), + ("te", "Telugu"), + ("tg", "Tajik"), + ("th", "Thai"), + ("ti", "Tigrinya"), + ("bo", "Tibetan"), + ("tk", "Turkmen"), + ("tl", "Tagalog"), + ("tn", "Tswana"), + ("to", "Tonga"), + ("tr", "Turkish"), + ("ts", "Tsonga"), + ("tt", "Tatar"), + ("tw", "Twi"), + ("ty", "Tahitian"), + ("ug", "Uighur, Uyghur"), + ("uk", "Ukrainian"), + ("ur", "Urdu"), + ("uz", "Uzbek"), + ("ve", "Venda"), + ("vi", "Vietnamese"), + ("vo", "Volapük"), + ("wa", "Walloon"), + ("cy", "Welsh"), + ("wo", "Wolof"), + ("fy", "Western Frisian"), + ("xh", "Xhosa"), + ("yi", "Yiddish"), + ("yo", "Yoruba"), + ("za", "Zhuang, Chuang"), + ("zu", "Zulu"), +] + + +ISO_639_BY_CODE = {code: name for code, name in ISO_639_CHOICES} diff --git a/api/funkwhale_api/common/models.py b/api/funkwhale_api/common/models.py index c10d8dd06..902fb0f5c 100644 --- a/api/funkwhale_api/common/models.py +++ b/api/funkwhale_api/common/models.py @@ -301,6 +301,20 @@ class Content(models.Model): return utils.render_html(self.text, self.content_type) + @property + def as_plain_text(self): + from . import utils + + return utils.render_plain_text(self.rendered) + + def truncate(self, length): + text = self.as_plain_text + truncated = text[:length] + if len(truncated) < len(text): + truncated += "…" + + return truncated + @receiver(models.signals.post_save, sender=Attachment) def warm_attachment_thumbnails(sender, instance, **kwargs): diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index d7135c4a0..51efce019 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -291,6 +291,10 @@ def render_html(text, content_type): return clean_html(rendered).strip().replace("\n", "") +def render_plain_text(html): + return bleach.clean(html, tags=[], strip=True) + + @transaction.atomic def attach_content(obj, field, content_data): from . import models diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 2592afb16..016e712c1 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -322,6 +322,10 @@ class Actor(models.Model): "https://{}/".format(domain) ) + @property + def display_name(self): + return self.name or self.preferred_username + FETCH_STATUSES = [ ("pending", "Pending"), diff --git a/api/funkwhale_api/subsonic/renderers.py b/api/funkwhale_api/subsonic/renderers.py index 527b3fa1e..ceab2c5ee 100644 --- a/api/funkwhale_api/subsonic/renderers.py +++ b/api/funkwhale_api/subsonic/renderers.py @@ -55,6 +55,8 @@ def dict_to_xml_tree(root_tag, d, parent=None): else: if key == "value": root.text = str(value) + elif key == "cdata_value": + root.text = "".format(str(value)) else: root.set(key, str(value)) return root diff --git a/api/tests/audio/test_serializers.py b/api/tests/audio/test_serializers.py index 243f52372..f0979c0e0 100644 --- a/api/tests/audio/test_serializers.py +++ b/api/tests/audio/test_serializers.py @@ -1,7 +1,13 @@ +import datetime + +import pytest +import pytz + from funkwhale_api.audio import serializers from funkwhale_api.common import serializers as common_serializers from funkwhale_api.common import utils as common_utils from funkwhale_api.federation import serializers as federation_serializers +from funkwhale_api.federation import utils as federation_utils from funkwhale_api.music import serializers as music_serializers @@ -43,6 +49,26 @@ def test_channel_serializer_create(factories): assert channel.library.actor == attributed_to +def test_channel_serializer_create_podcast(factories): + attributed_to = factories["federation.Actor"](local=True) + + data = { + # TODO: cover + "name": "My channel", + "username": "mychannel", + "description": {"text": "This is my channel", "content_type": "text/markdown"}, + "tags": ["hello", "world"], + "content_category": "podcast", + "metadata": {"itunes_category": "Sports", "language": "en"}, + } + + serializer = serializers.ChannelCreateSerializer(data=data) + assert serializer.is_valid(raise_exception=True) is True + + channel = serializer.save(attributed_to=attributed_to) + assert channel.metadata == data["metadata"] + + def test_channel_serializer_update(factories): channel = factories["audio.Channel"](artist__set_tags=["rock"]) @@ -74,6 +100,27 @@ def test_channel_serializer_update(factories): assert channel.actor.name == data["name"] +def test_channel_serializer_update_podcast(factories): + channel = factories["audio.Channel"](artist__set_tags=["rock"]) + + data = { + # TODO: cover + "name": "My channel", + "description": {"text": "This is my channel", "content_type": "text/markdown"}, + "tags": ["hello", "world"], + "content_category": "podcast", + "metadata": {"language": "en", "itunes_category": "Sports"}, + } + + serializer = serializers.ChannelUpdateSerializer(channel, data=data) + assert serializer.is_valid(raise_exception=True) is True + + serializer.save() + channel.refresh_from_db() + + assert channel.metadata == data["metadata"] + + def test_channel_serializer_representation(factories, to_api_date): content = factories["common.Content"]() channel = factories["audio.Channel"](artist__description=content) @@ -86,6 +133,7 @@ def test_channel_serializer_representation(factories, to_api_date): "attributed_to": federation_serializers.APIActorSerializer( channel.attributed_to ).data, + "metadata": {}, } expected["artist"]["description"] = common_serializers.ContentSerializer( content @@ -115,3 +163,169 @@ def test_subscription_serializer(factories, to_api_date): } assert serializers.SubscriptionSerializer(subscription).data == expected + + +def test_rss_item_serializer(factories): + description = factories["common.Content"]() + upload = factories["music.Upload"]( + playable=True, + track__set_tags=["pop", "rock"], + track__description=description, + track__disc_number=4, + track__position=42, + ) + setattr( + upload.track, + "_prefetched_tagged_items", + upload.track.tagged_items.order_by("tag__name"), + ) + expected = { + "title": [{"value": upload.track.title}], + "itunes:title": [{"value": upload.track.title}], + "itunes:subtitle": [{"value": description.truncate(255)}], + "itunes:summary": [{"cdata_value": description.rendered}], + "description": [{"value": description.as_plain_text}], + "content:encoded": [{"cdata_value": description.rendered}], + "guid": [{"cdata_value": str(upload.uuid), "isPermalink": "false"}], + "pubDate": [{"value": serializers.rss_date(upload.creation_date)}], + "itunes:duration": [{"value": serializers.rss_duration(upload.duration)}], + "itunes:keywords": [{"value": "pop rock"}], + "itunes:explicit": [{"value": "no"}], + "itunes:episodeType": [{"value": "full"}], + "itunes:season": [{"value": upload.track.disc_number}], + "itunes:episode": [{"value": upload.track.position}], + "itunes:image": [{"href": upload.track.attachment_cover.download_url_original}], + "link": [{"value": federation_utils.full_url(upload.track.get_absolute_url())}], + "enclosure": [ + {"url": upload.listen_url, "length": upload.size, "type": upload.mimetype} + ], + } + + assert serializers.rss_serialize_item(upload) == expected + + +def test_rss_channel_serializer(factories): + metadata = { + "language": "fr", + "itunes_category": "Parent", + "itunes_subcategory": "Child", + "copyright": "Myself", + "owner_name": "Name", + "owner_email": "name@domain.com", + "explicit": True, + } + description = factories["common.Content"]() + channel = factories["audio.Channel"]( + artist__set_tags=["pop", "rock"], + artist__description=description, + metadata=metadata, + ) + setattr( + channel.artist, + "_prefetched_tagged_items", + channel.artist.tagged_items.order_by("tag__name"), + ) + + expected = { + "title": [{"value": channel.artist.name}], + "language": [{"value": metadata["language"]}], + "copyright": [{"value": metadata["copyright"]}], + "itunes:subtitle": [{"value": description.truncate(255)}], + "itunes:summary": [{"cdata_value": description.rendered}], + "description": [{"value": description.as_plain_text}], + "itunes:keywords": [{"value": "pop rock"}], + "itunes:category": [ + { + "text": metadata["itunes_category"], + "itunes:category": [{"text": metadata["itunes_subcategory"]}], + } + ], + "itunes:explicit": [{"value": "yes"}], + "itunes:owner": [ + { + "itunes:name": [{"value": metadata["owner_name"]}], + "itunes:email": [{"value": metadata["owner_email"]}], + } + ], + "itunes:author": [{"value": metadata["owner_name"]}], + "itunes:type": [{"value": "episodic"}], + "itunes:image": [ + {"href": channel.artist.attachment_cover.download_url_original} + ], + "link": [{"value": channel.get_absolute_url()}], + "atom:link": [ + { + "href": channel.get_rss_url(), + "rel": "self", + "type": "application/rss+xml", + } + ], + } + + assert serializers.rss_serialize_channel(channel) == expected + + +def test_serialize_full_channel(factories): + channel = factories["audio.Channel"]() + upload1 = factories["music.Upload"](playable=True) + upload2 = factories["music.Upload"](playable=True) + + expected = serializers.rss_serialize_channel(channel) + expected["item"] = [ + serializers.rss_serialize_item(upload1), + serializers.rss_serialize_item(upload2), + ] + expected = {"channel": expected} + + result = serializers.rss_serialize_channel_full( + channel=channel, uploads=[upload1, upload2] + ) + + assert result == expected + + +@pytest.mark.parametrize( + "seconds, expected", + [ + (0, "00:00:00"), + (None, "00:00:00"), + (61, "00:01:01"), + (3601, "01:00:01"), + (7345, "02:02:25"), + ], +) +def test_rss_duration(seconds, expected): + assert serializers.rss_duration(seconds) == expected + + +@pytest.mark.parametrize( + "dt, expected", + [ + ( + datetime.datetime(2020, 1, 30, 6, 0, 49, tzinfo=pytz.UTC), + "Thu, 30 Jan 2020 06:00:49 +0000", + ), + ], +) +def test_rss_date(dt, expected): + assert serializers.rss_date(dt) == expected + + +def test_channel_metadata_serializer_validation(): + payload = { + "language": "fr", + "copyright": "Me", + "owner_email": "contact@me.com", + "owner_name": "Me", + "itunes_category": "Health & Fitness", + "itunes_subcategory": "Sexuality", + "unknown_key": "noop", + } + + serializer = serializers.ChannelMetadataSerializer(data=payload) + + assert serializer.is_valid(raise_exception=True) is True + + payload.pop("unknown_key") + + assert serializer.validated_data == payload diff --git a/api/tests/audio/test_spa_views.py b/api/tests/audio/test_spa_views.py index bae96e711..9bb96807a 100644 --- a/api/tests/audio/test_spa_views.py +++ b/api/tests/audio/test_spa_views.py @@ -33,6 +33,13 @@ def test_library_artist(spa_html, no_api_auth, client, factories, settings): "type": "application/activity+json", "href": channel.actor.fid, }, + { + "tag": "link", + "rel": "alternate", + "type": "application/rss+xml", + "href": channel.get_rss_url(), + "title": "{} - RSS Podcast Feed".format(channel.artist.name), + }, { "tag": "link", "rel": "alternate", diff --git a/api/tests/audio/test_views.py b/api/tests/audio/test_views.py index b40fa77bf..7a9fb477b 100644 --- a/api/tests/audio/test_views.py +++ b/api/tests/audio/test_views.py @@ -15,6 +15,7 @@ def test_channel_create(logged_in_api_client): "description": {"text": "This is my channel", "content_type": "text/markdown"}, "tags": ["hello", "world"], "content_category": "podcast", + "metadata": {"language": "en", "itunes_category": "Sports"}, } url = reverse("api:v1:channels-list") @@ -192,3 +193,21 @@ def test_subscriptions_all(factories, logged_in_api_client): assert response.status_code == 200 assert response.data == {"results": [subscription.uuid], "count": 1} + + +def test_channel_rss_feed(factories, api_client): + channel = factories["audio.Channel"]() + upload1 = factories["music.Upload"](library=channel.library, playable=True) + upload2 = factories["music.Upload"](library=channel.library, playable=True) + + expected = serializers.rss_serialize_channel_full( + channel=channel, uploads=[upload2, upload1] + ) + + url = reverse("api:v1:channels-rss", kwargs={"uuid": channel.uuid}) + + response = api_client.get(url) + + assert response.status_code == 200 + assert response.data == expected + assert response["Content-Type"] == "application/rss+xml" diff --git a/api/tests/common/test_models.py b/api/tests/common/test_models.py index f60ee3ef5..42992095f 100644 --- a/api/tests/common/test_models.py +++ b/api/tests/common/test_models.py @@ -85,3 +85,19 @@ def test_removing_obj_removes_content(factories): removed_content.refresh_from_db() kept_content.refresh_from_db() + + +def test_content_as_plain_text(factories): + content = factories["common.Content"]( + content_type="text/html", text="hello world" + ) + + assert content.as_plain_text == "hello world" + + +def test_content_truncate(factories): + content = factories["common.Content"]( + content_type="text/html", text="hello world" + ) + + assert content.truncate(5) == "hello…" diff --git a/docker/traefik.toml b/docker/traefik.toml index c87f4527d..96641316c 100644 --- a/docker/traefik.toml +++ b/docker/traefik.toml @@ -16,8 +16,8 @@ exposedbydefault = false [entryPoints] [entryPoints.http] address = ":80" - [entryPoints.http.redirect] - entryPoint = "https" + # [entryPoints.http.redirect] + entryPoint = "http" [entryPoints.https] address = ":443" [entryPoints.https.tls]