mirror
/
funkwhale
kopia lustrzana https://dev.funkwhale.audio/funkwhale/funkwhale
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

See #170: subscribe to 3d-party RSS feeds in Funkwhale

environments/review-front-list-6rg6z1/deployments/4496
Eliot Berriot 2020-03-13 12:16:51 +01:00
rodzic 7cae1ae5db
commit deb1f35779
29 zmienionych plików z 1451 dodań i 129 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

13
api/config/settings/common.py
Wyświetl plik

@ -580,6 +580,11 @@ CELERY_BROKER_URL = env(
CELERY_TASK_DEFAULT_RATE_LIMIT = 1
CELERY_TASK_TIME_LIMIT = 300
CELERY_BEAT_SCHEDULE = {
"audio.fetch_rss_feeds": {
"task": "audio.fetch_rss_feeds",
"schedule": crontab(minute="0", hour="*"),
"options": {"expires": 60 * 60},
},
"common.prune_unattached_attachments": {
"task": "common.prune_unattached_attachments",
"schedule": crontab(minute="0", hour="*"),
@ -976,3 +981,11 @@ MIN_DELAY_BETWEEN_DOWNLOADS_COUNT = env.int(
MARKDOWN_EXTENSIONS = env.list("MARKDOWN_EXTENSIONS", default=["nl2br", "extra"])
LINKIFIER_SUPPORTED_TLDS = ["audio"] + env.list("LINKINFIER_SUPPORTED_TLDS", default=[])
EXTERNAL_MEDIA_PROXY_ENABLED = env.bool("EXTERNAL_MEDIA_PROXY_ENABLED", default=True)
# By default, only people who subscribe to a podcast RSS will have access to it
# switch to "instance" or "everyone" to change that
PODCASTS_THIRD_PARTY_VISIBILITY = env("PODCASTS_THIRD_PARTY_VISIBILITY", default="me")
PODCASTS_RSS_FEED_REFRESH_DELAY = env.int(
"PODCASTS_RSS_FEED_REFRESH_DELAY", default=60 * 60 * 24
)

2
api/funkwhale_api/audio/categories.py
Wyświetl plik

@ -109,3 +109,5 @@ ITUNES_CATEGORIES = {
"TV Reviews",
],
}
ITUNES_SUBCATEGORIES = [s for p in ITUNES_CATEGORIES.values() for s in p]

17
api/funkwhale_api/audio/factories.py
Wyświetl plik

@ -1,6 +1,9 @@
import uuid
import factory
from funkwhale_api.factories import registry, NoUpdateOnCreate
from funkwhale_api.federation import actors
from funkwhale_api.federation import factories as federation_factories
from funkwhale_api.music import factories as music_factories
@ -11,6 +14,10 @@ def set_actor(o):
return models.generate_actor(str(o.uuid))
def get_rss_channel_name():
return "rssfeed-{}".format(uuid.uuid4())
@registry.register
class ChannelFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
uuid = factory.Faker("uuid4")
@ -32,10 +39,20 @@ class ChannelFactory(NoUpdateOnCreate, factory.django.DjangoModelFactory):
model = "audio.Channel"
class Params:
external = factory.Trait(
attributed_to=factory.LazyFunction(actors.get_service_actor),
library__privacy_level="me",
actor=factory.SubFactory(
federation_factories.ActorFactory,
local=True,
preferred_username=factory.LazyFunction(get_rss_channel_name),
),
)
local = factory.Trait(
attributed_to=factory.SubFactory(
federation_factories.ActorFactory, local=True
),
library__privacy_level="everyone",
artist__local=True,
)

22
api/funkwhale_api/audio/models.py
Wyświetl plik

@ -19,6 +19,19 @@ def empty_dict():
return {}
class ChannelQuerySet(models.QuerySet):
def external_rss(self, include=True):
from funkwhale_api.federation import actors
query = models.Q(
attributed_to=actors.get_service_actor(),
actor__preferred_username__startswith="rssfeed-",
)
if include:
return self.filter(query)
return self.exclude(query)
class Channel(models.Model):
uuid = models.UUIDField(default=uuid.uuid4, unique=True)
artist = models.OneToOneField(
@ -45,6 +58,8 @@ class Channel(models.Model):
default=empty_dict, max_length=50000, encoder=DjangoJSONEncoder, blank=True
)
objects = ChannelQuerySet.as_manager()
def get_absolute_url(self):
suffix = self.uuid
if self.actor.is_local:
@ -54,7 +69,9 @@ class Channel(models.Model):
return federation_utils.full_url("/channels/{}".format(suffix))
def get_rss_url(self):
if not self.artist.is_local:
if not self.artist.is_local or self.actor.preferred_username.startswith(
"rssfeed-"
):
return self.rss_url
return federation_utils.full_url(
@ -81,5 +98,6 @@ def generate_actor(username, **kwargs):
@receiver(post_delete, sender=Channel)
def delete_channel_related_objs(instance, **kwargs):
instance.library.delete()
instance.actor.delete()
if instance.actor != instance.attributed_to:
instance.actor.delete()
instance.artist.delete()

10
api/funkwhale_api/audio/renderers.py
Wyświetl plik

@ -21,12 +21,16 @@ class PodcastRSSRenderer(renderers.JSONRenderer):
}
final.update(data)
tree = dict_to_xml_tree("rss", final)
return b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(
tree, encoding="utf-8"
)
return render_xml(tree)
class PodcastRSSContentNegociation(negotiation.DefaultContentNegotiation):
def select_renderer(self, request, renderers, format_suffix=None):
return (PodcastRSSRenderer(), PodcastRSSRenderer.media_type)
def render_xml(tree):
return b'<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(
tree, encoding="utf-8"
)

496
api/funkwhale_api/audio/serializers.py
Wyświetl plik

@ -1,17 +1,32 @@
import datetime
import logging
import time
import uuid
from django.conf import settings
from django.db import transaction
from django.db.models import Q
from django.utils import timezone
import feedparser
import requests
import pytz
from rest_framework import serializers
from django.templatetags.static import static
from django.urls import reverse
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.common import utils as common_utils
from funkwhale_api.common import locales
from funkwhale_api.common import preferences
from funkwhale_api.common import session
from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.moderation import mrf
from funkwhale_api.music import models as music_models
from funkwhale_api.music import serializers as music_serializers
from funkwhale_api.tags import models as tags_models
@ -22,6 +37,9 @@ from . import categories
from . import models
logger = logging.getLogger(__name__)
class ChannelMetadataSerializer(serializers.Serializer):
itunes_category = serializers.ChoiceField(
choices=categories.ITUNES_CATEGORIES, required=True
@ -218,7 +236,7 @@ class ChannelUpdateSerializer(serializers.Serializer):
class ChannelSerializer(serializers.ModelSerializer):
artist = serializers.SerializerMethodField()
actor = federation_serializers.APIActorSerializer()
actor = serializers.SerializerMethodField()
attributed_to = federation_serializers.APIActorSerializer()
rss_url = serializers.CharField(source="get_rss_url")
@ -246,6 +264,11 @@ class ChannelSerializer(serializers.ModelSerializer):
def get_subscriptions_count(self, obj):
return obj.actor.received_follows.exclude(approved=False).count()
def get_actor(self, obj):
if obj.attributed_to == actors.get_service_actor():
return None
return federation_serializers.APIActorSerializer(obj.actor).data
class SubscriptionSerializer(serializers.Serializer):
approved = serializers.BooleanField(read_only=True)
@ -259,11 +282,475 @@ class SubscriptionSerializer(serializers.Serializer):
return data
class RssSubscribeSerializer(serializers.Serializer):
url = serializers.URLField()
class FeedFetchException(Exception):
pass
class BlockedFeedException(FeedFetchException):
pass
def retrieve_feed(url):
try:
logger.info("Fetching RSS feed at %s", url)
response = session.get_session().get(url)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response:
raise FeedFetchException(
"Error while fetching feed: HTTP {}".format(e.response.status_code)
)
raise FeedFetchException("Error while fetching feed: unknown error")
except requests.exceptions.Timeout:
raise FeedFetchException("Error while fetching feed: timeout")
except requests.exceptions.ConnectionError:
raise FeedFetchException("Error while fetching feed: connection error")
except requests.RequestException as e:
raise FeedFetchException("Error while fetching feed: {}".format(e))
except Exception as e:
raise FeedFetchException("Error while fetching feed: {}".format(e))
return response
@transaction.atomic
def get_channel_from_rss_url(url):
# first, check if the url is blocked
is_valid, _ = mrf.inbox.apply({"id": url})
if not is_valid:
logger.warn("Feed fetch for url %s dropped by MRF", url)
raise BlockedFeedException("This feed or domain is blocked")
# retrieve the XML payload at the given URL
response = retrieve_feed(url)
parsed_feed = feedparser.parse(response.text)
serializer = RssFeedSerializer(data=parsed_feed["feed"])
if not serializer.is_valid():
raise FeedFetchException("Invalid xml content: {}".format(serializer.errors))
# second mrf check with validated data
urls_to_check = set()
atom_link = serializer.validated_data.get("atom_link")
if atom_link and atom_link != url:
urls_to_check.add(atom_link)
if serializer.validated_data["link"] != url:
urls_to_check.add(serializer.validated_data["link"])
for u in urls_to_check:
is_valid, _ = mrf.inbox.apply({"id": u})
if not is_valid:
logger.warn("Feed fetch for url %s dropped by MRF", u)
raise BlockedFeedException("This feed or domain is blocked")
# now, we're clear, we can save the data
channel = serializer.save(rss_url=url)
entries = parsed_feed.entries or []
uploads = []
track_defaults = {}
existing_uploads = list(
channel.library.uploads.all().select_related(
"track__description", "track__attachment_cover"
)
)
if parsed_feed.feed.rights:
track_defaults["copyright"] = parsed_feed.feed.rights
for entry in entries:
logger.debug("Importing feed item %s", entry.id)
s = RssFeedItemSerializer(data=entry)
if not s.is_valid():
logger.debug("Skipping invalid RSS feed item %s", entry)
continue
uploads.append(
s.save(channel, existing_uploads=existing_uploads, **track_defaults)
)
common_utils.on_commit(
music_models.TrackActor.create_entries,
library=channel.library,
delete_existing=True,
)
return channel, uploads
# RSS related stuff
# https://github.com/simplepie/simplepie-ng/wiki/Spec:-iTunes-Podcast-RSS
# is extremely useful
class RssFeedSerializer(serializers.Serializer):
title = serializers.CharField()
link = serializers.URLField()
language = serializers.CharField(required=False, allow_blank=True)
rights = serializers.CharField(required=False, allow_blank=True)
itunes_explicit = serializers.BooleanField(required=False, allow_null=True)
tags = serializers.ListField(required=False)
atom_link = serializers.DictField(required=False)
summary_detail = serializers.DictField(required=False)
author_detail = serializers.DictField(required=False)
image = serializers.DictField(required=False)
def validate_atom_link(self, v):
if (
v.get("rel", "self") == "self"
and v.get("type", "application/rss+xml") == "application/rss+xml"
):
return v["href"]
def validate_summary_detail(self, v):
content = v.get("value")
if not content:
return
return {
"content_type": v.get("type", "text/plain"),
"text": content,
}
def validate_image(self, v):
url = v.get("href")
if url:
return {
"url": url,
"mimetype": common_utils.get_mimetype_from_ext(url) or "image/jpeg",
}
def validate_tags(self, v):
data = {}
for row in v:
if row.get("scheme") != "http://www.itunes.com/":
continue
term = row["term"]
if "parent" not in data and term in categories.ITUNES_CATEGORIES:
data["parent"] = term
elif "child" not in data and term in categories.ITUNES_SUBCATEGORIES:
data["child"] = term
elif (
term not in categories.ITUNES_SUBCATEGORIES
and term not in categories.ITUNES_CATEGORIES
):
raw_tags = term.split(" ")
data["tags"] = []
tag_serializer = tags_serializers.TagNameField()
for tag in raw_tags:
try:
data["tags"].append(tag_serializer.to_internal_value(tag))
except Exception:
pass
return data
@transaction.atomic
def save(self, rss_url):
validated_data = self.validated_data
# because there may be redirections from the original feed URL
real_rss_url = validated_data.get("atom_link", rss_url) or rss_url
service_actor = actors.get_service_actor()
author = validated_data.get("author_detail", {})
categories = validated_data.get("tags", {})
metadata = {
"explicit": validated_data.get("itunes_explicit", False),
"copyright": validated_data.get("rights"),
"owner_name": author.get("name"),
"owner_email": author.get("email"),
"itunes_category": categories.get("parent"),
"itunes_subcategory": categories.get("child"),
"language": validated_data.get("language"),
}
public_url = validated_data["link"]
existing = (
models.Channel.objects.external_rss()
.filter(
Q(rss_url=real_rss_url) | Q(rss_url=rss_url) | Q(actor__url=public_url)
)
.first()
)
channel_defaults = {
"rss_url": real_rss_url,
"metadata": metadata,
}
if existing:
artist_kwargs = {"channel": existing}
actor_kwargs = {"channel": existing}
actor_defaults = {"url": public_url}
else:
artist_kwargs = {"pk": None}
actor_kwargs = {"pk": None}
preferred_username = "rssfeed-{}".format(uuid.uuid4())
actor_defaults = {
"preferred_username": preferred_username,
"type": "Application",
"domain": service_actor.domain,
"url": public_url,
"fid": federation_utils.full_url(
reverse(
"federation:actors-detail",
kwargs={"preferred_username": preferred_username},
)
),
}
channel_defaults["attributed_to"] = service_actor
actor_defaults["last_fetch_date"] = timezone.now()
# create/update the artist profile
artist, created = music_models.Artist.objects.update_or_create(
**artist_kwargs,
defaults={
"attributed_to": service_actor,
"name": validated_data["title"],
"content_category": "podcast",
},
)
cover = validated_data.get("image")
if cover:
common_utils.attach_file(artist, "attachment_cover", cover)
tags = categories.get("tags", [])
if tags:
tags_models.set_tags(artist, *tags)
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(artist, "description", summary)
if created:
channel_defaults["artist"] = artist
# create/update the actor
actor, created = federation_models.Actor.objects.update_or_create(
**actor_kwargs, defaults=actor_defaults
)
if created:
channel_defaults["actor"] = actor
# create the library
if not existing:
channel_defaults["library"] = music_models.Library.objects.create(
actor=service_actor,
privacy_level=settings.PODCASTS_THIRD_PARTY_VISIBILITY,
name=actor_defaults["preferred_username"],
)
# create/update the channel
channel, created = models.Channel.objects.update_or_create(
pk=existing.pk if existing else None, defaults=channel_defaults,
)
return channel
class ItunesDurationField(serializers.CharField):
def to_internal_value(self, v):
try:
return int(v)
except (ValueError, TypeError):
pass
parts = v.split(":")
int_parts = []
for part in parts:
try:
int_parts.append(int(part))
except (ValueError, TypeError):
raise serializers.ValidationError("Invalid duration {}".format(v))
if len(int_parts) == 2:
hours = 0
minutes, seconds = int_parts
elif len(int_parts) == 3:
hours, minutes, seconds = int_parts
else:
raise serializers.ValidationError("Invalid duration {}".format(v))
return (hours * 3600) + (minutes * 60) + seconds
class DummyField(serializers.Field):
def to_internal_value(self, v):
return v
def get_cached_upload(uploads, expected_track_uuid):
for upload in uploads:
if upload.track.uuid == expected_track_uuid:
return upload
class RssFeedItemSerializer(serializers.Serializer):
id = serializers.CharField()
title = serializers.CharField()
rights = serializers.CharField(required=False, allow_blank=True)
itunes_season = serializers.IntegerField(required=False)
itunes_episode = serializers.IntegerField(required=False)
itunes_duration = ItunesDurationField()
links = serializers.ListField()
tags = serializers.ListField(required=False)
summary_detail = serializers.DictField(required=False)
published_parsed = DummyField(required=False)
image = serializers.DictField(required=False)
def validate_summary_detail(self, v):
content = v.get("value")
if not content:
return
return {
"content_type": v.get("type", "text/plain"),
"text": content,
}
def validate_image(self, v):
url = v.get("href")
if url:
return {
"url": url,
"mimetype": common_utils.get_mimetype_from_ext(url) or "image/jpeg",
}
def validate_links(self, v):
data = {}
for row in v:
if not row.get("type", "").startswith("audio/"):
continue
if row.get("rel") != "enclosure":
continue
try:
size = int(row.get("length"))
except (TypeError, ValueError):
raise serializers.ValidationError("Invalid size")
data["audio"] = {
"mimetype": row["type"],
"size": size,
"source": row["href"],
}
if not data:
raise serializers.ValidationError("No valid audio enclosure found")
return data
def validate_tags(self, v):
data = {}
for row in v:
if row.get("scheme") != "http://www.itunes.com/":
continue
term = row["term"]
raw_tags = term.split(" ")
data["tags"] = []
tag_serializer = tags_serializers.TagNameField()
for tag in raw_tags:
try:
data["tags"].append(tag_serializer.to_internal_value(tag))
except Exception:
pass
return data
@transaction.atomic
def save(self, channel, existing_uploads=[], **track_defaults):
validated_data = self.validated_data
categories = validated_data.get("tags", {})
expected_uuid = uuid.uuid3(
uuid.NAMESPACE_URL, "rss://{}-{}".format(channel.pk, validated_data["id"])
)
existing_upload = get_cached_upload(existing_uploads, expected_uuid)
if existing_upload:
existing_track = existing_upload.track
else:
existing_track = (
music_models.Track.objects.filter(
uuid=expected_uuid, artist__channel=channel
)
.select_related("description", "attachment_cover")
.first()
)
if existing_track:
existing_upload = existing_track.uploads.filter(
library=channel.library
).first()
track_defaults = track_defaults
track_defaults.update(
{
"disc_number": validated_data.get("itunes_season", 1),
"position": validated_data.get("itunes_episode", 1),
"title": validated_data["title"],
"artist": channel.artist,
}
)
if "rights" in validated_data:
track_defaults["rights"] = validated_data["rights"]
if "published_parsed" in validated_data:
track_defaults["creation_date"] = datetime.datetime.fromtimestamp(
time.mktime(validated_data["published_parsed"])
).replace(tzinfo=pytz.utc)
upload_defaults = {
"source": validated_data["links"]["audio"]["source"],
"size": validated_data["links"]["audio"]["size"],
"mimetype": validated_data["links"]["audio"]["mimetype"],
"duration": validated_data["itunes_duration"],
"import_status": "finished",
"library": channel.library,
}
if existing_track:
track_kwargs = {"pk": existing_track.pk}
upload_kwargs = {"track": existing_track}
else:
track_kwargs = {"pk": None}
track_defaults["uuid"] = expected_uuid
upload_kwargs = {"pk": None}
if existing_upload and existing_upload.source != upload_defaults["source"]:
# delete existing upload, the url to the audio file has changed
existing_upload.delete()
# create/update the track
track, created = music_models.Track.objects.update_or_create(
**track_kwargs, defaults=track_defaults,
)
# optimisation for reducing SQL queries, because we cannot use select_related with
# update or create, so we restore the cache by hand
if existing_track:
for field in ["attachment_cover", "description"]:
cached_id_value = getattr(existing_track, "{}_id".format(field))
new_id_value = getattr(track, "{}_id".format(field))
if new_id_value and cached_id_value == new_id_value:
setattr(track, field, getattr(existing_track, field))
cover = validated_data.get("image")
if cover:
common_utils.attach_file(track, "attachment_cover", cover)
tags = categories.get("tags", [])
if tags:
tags_models.set_tags(track, *tags)
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(track, "description", summary)
if created:
upload_defaults["track"] = track
# create/update the upload
upload, created = music_models.Upload.objects.update_or_create(
**upload_kwargs, defaults=upload_defaults
)
return upload
def rss_date(dt):
return dt.strftime("%a, %d %b %Y %H:%M:%S %z")
@ -344,7 +831,12 @@ def rss_serialize_channel(channel):
"href": channel.get_rss_url(),
"rel": "self",
"type": "application/rss+xml",
}
},
{
"href": channel.actor.fid,
"rel": "alternate",
"type": "application/activity+json",
},
],
}
if language:

51
api/funkwhale_api/audio/tasks.py 100644
Wyświetl plik

@ -0,0 +1,51 @@
import datetime
import logging
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from funkwhale_api.taskapp import celery
from . import models
from . import serializers
logger = logging.getLogger(__name__)
@celery.app.task(name="audio.fetch_rss_feeds")
def fetch_rss_feeds():
limit = timezone.now() - datetime.timedelta(
seconds=settings.PODCASTS_RSS_FEED_REFRESH_DELAY
)
candidates = (
models.Channel.objects.external_rss()
.filter(actor__last_fetch_date__lte=limit)
.values_list("rss_url", flat=True)
)
total = len(candidates)
logger.info("Refreshing %s rss feeds…", total)
for url in candidates:
fetch_rss_feed.delay(rss_url=url)
@celery.app.task(name="audio.fetch_rss_feed")
@transaction.atomic
def fetch_rss_feed(rss_url):
channel = (
models.Channel.objects.external_rss()
.filter(rss_url=rss_url)
.order_by("id")
.first()
)
if not channel:
logger.warn("Cannot refresh non external feed")
return
try:
serializers.get_channel_from_rss_url(rss_url)
except serializers.BlockedFeedException:
# channel was blocked since last fetch, let's delete it
logger.info("Deleting blocked channel linked to %s", rss_url)
channel.delete()

71
api/funkwhale_api/audio/views.py
Wyświetl plik

@ -8,12 +8,12 @@ from rest_framework import viewsets
from django import http
from django.db import transaction
from django.db.models import Count, Prefetch, Q
from django.db.utils import IntegrityError
from funkwhale_api.common import locales
from funkwhale_api.common import permissions
from funkwhale_api.common import preferences
from funkwhale_api.common.mixins import MultipleLookupDetailMixin
from funkwhale_api.federation import actors
from funkwhale_api.federation import models as federation_models
from funkwhale_api.federation import routes
from funkwhale_api.federation import utils as federation_utils
@ -100,17 +100,19 @@ class ChannelViewSet(
)
def subscribe(self, request, *args, **kwargs):
object = self.get_object()
subscription = federation_models.Follow(
target=object.actor, approved=True, actor=request.user.actor,
)
subscription = federation_models.Follow(actor=request.user.actor)
subscription.fid = subscription.get_federation_id()
try:
subscription.save()
except IntegrityError:
# there's already a subscription for this actor/channel
subscription = object.actor.received_follows.filter(
actor=request.user.actor
).get()
subscription, created = SubscriptionsViewSet.queryset.get_or_create(
target=object.actor,
actor=request.user.actor,
defaults={
"approved": True,
"fid": subscription.fid,
"uuid": subscription.uuid,
},
)
# prefetch stuff
subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
data = serializers.SubscriptionSerializer(subscription).data
return response.Response(data, status=201)
@ -135,6 +137,10 @@ class ChannelViewSet(
if not object.attributed_to.is_local:
return response.Response({"detail": "Not found"}, status=404)
if object.attributed_to == actors.get_service_actor():
# external feed, we redirect to the canonical one
return http.HttpResponseRedirect(object.rss_url)
uploads = (
object.library.uploads.playable_by(None)
.prefetch_related(
@ -170,6 +176,49 @@ class ChannelViewSet(
}
return response.Response(data)
@decorators.action(
methods=["post"],
detail=False,
url_path="rss-subscribe",
url_name="rss_subscribe",
)
@transaction.atomic
def rss_subscribe(self, request, *args, **kwargs):
serializer = serializers.RssSubscribeSerializer(data=request.data)
if not serializer.is_valid():
return response.Response(serializer.errors, status=400)
channel = (
models.Channel.objects.filter(rss_url=serializer.validated_data["url"],)
.order_by("id")
.first()
)
if not channel:
# try to retrieve the channel via its URL and create it
try:
channel, uploads = serializers.get_channel_from_rss_url(
serializer.validated_data["url"]
)
except serializers.FeedFetchException as e:
return response.Response({"detail": str(e)}, status=400,)
subscription = federation_models.Follow(actor=request.user.actor)
subscription.fid = subscription.get_federation_id()
subscription, created = SubscriptionsViewSet.queryset.get_or_create(
target=channel.actor,
actor=request.user.actor,
defaults={
"approved": True,
"fid": subscription.fid,
"uuid": subscription.uuid,
},
)
# prefetch stuff
subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
return response.Response(
serializers.SubscriptionSerializer(subscription).data, status=201
)
def get_serializer_context(self):
context = super().get_serializer_context()
context["subscriptions_count"] = self.action in [

20
api/funkwhale_api/common/utils.py
Wyświetl plik

@ -310,13 +310,21 @@ def render_plain_text(html):
return bleach.clean(html, tags=[], strip=True)
def same_content(old, text=None, content_type=None):
return old.text == text and old.content_type == content_type
@transaction.atomic
def attach_content(obj, field, content_data):
from . import models
content_data = content_data or {}
existing = getattr(obj, "{}_id".format(field))
if existing:
if same_content(getattr(obj, field), **content_data):
# optimization to avoid a delete/save if possible
return getattr(obj, field)
getattr(obj, field).delete()
setattr(obj, field, None)
@ -376,3 +384,15 @@ def attach_file(obj, field, file_data, fetch=False):
setattr(obj, field, attachment)
obj.save(update_fields=[field])
return attachment
def get_mimetype_from_ext(path):
parts = path.lower().split(".")
ext = parts[-1]
match = {
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
}
return match.get(ext)

4
api/funkwhale_api/common/views.py
Wyświetl plik

@ -163,6 +163,10 @@ class AttachmentViewSet(
@transaction.atomic
def proxy(self, request, *args, **kwargs):
instance = self.get_object()
if not settings.EXTERNAL_MEDIA_PROXY_ENABLED:
r = response.Response(status=302)
r["Location"] = instance.url
return r
size = request.GET.get("next", "original").lower()
if size not in ["original", "medium_square_crop"]:

17
api/funkwhale_api/federation/actors.py
Wyświetl plik

@ -42,21 +42,32 @@ def get_actor(fid, skip_cache=False):
return serializer.save(last_fetch_date=timezone.now())
def get_service_actor():
_CACHE = {}
def get_service_actor(cache=True):
if cache and "service_actor" in _CACHE:
return _CACHE["service_actor"]
name, domain = (
settings.FEDERATION_SERVICE_ACTOR_USERNAME,
settings.FEDERATION_HOSTNAME,
)
try:
return models.Actor.objects.select_related().get(
actor = models.Actor.objects.select_related().get(
preferred_username=name, domain__name=domain
)
except models.Actor.DoesNotExist:
pass
else:
_CACHE["service_actor"] = actor
return actor
args = users_models.get_actor_data(name)
private, public = keys.get_key_pair()
args["private_key"] = private.decode("utf-8")
args["public_key"] = public.decode("utf-8")
args["type"] = "Service"
return models.Actor.objects.create(**args)
actor = models.Actor.objects.create(**args)
_CACHE["service_actor"] = actor
return actor

1
api/funkwhale_api/federation/tasks.py
Wyświetl plik

@ -311,6 +311,7 @@ def fetch(fetch_obj):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
else:
auth = None
auth = None
try:
if url.startswith("webfinger://"):
# we first grab the correpsonding webfinger representation

15
api/funkwhale_api/federation/views.py
Wyświetl plik

@ -13,7 +13,16 @@ from funkwhale_api.moderation import models as moderation_models
from funkwhale_api.music import models as music_models
from funkwhale_api.music import utils as music_utils
from . import activity, authentication, models, renderers, serializers, utils, webfinger
from . import (
actors,
activity,
authentication,
models,
renderers,
serializers,
utils,
webfinger,
)
def redirect_to_html(public_url):
@ -61,6 +70,10 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV
queryset = models.Actor.objects.local().select_related("user")
serializer_class = serializers.ActorSerializer
def get_queryset(self):
queryset = super().get_queryset()
return queryset.exclude(channel__attributed_to=actors.get_service_actor())
def retrieve(self, request, *args, **kwargs):
instance = self.get_object()
if utils.should_redirect_ap_to_html(request.headers.get("accept")):

7
api/funkwhale_api/music/admin.py
Wyświetl plik

@ -23,6 +23,13 @@ class TrackAdmin(admin.ModelAdmin):
list_select_related = ["album__artist", "artist"]
@admin.register(models.TrackActor)
class TrackActorAdmin(admin.ModelAdmin):
list_display = ["actor", "track", "upload", "internal"]
search_fields = ["actor__preferred_username", "track__name"]
list_select_related = ["actor", "track"]
@admin.register(models.ImportBatch)
class ImportBatchAdmin(admin.ModelAdmin):
list_display = ["submitted_by", "creation_date", "import_request", "status"]

26
api/funkwhale_api/music/models.py
Wyświetl plik

@ -786,9 +786,13 @@ class Upload(models.Model):
with remote_response as r:
remote_response.raise_for_status()
extension = utils.get_ext_from_type(self.mimetype)
title = " - ".join(
[self.track.title, self.track.album.title, self.track.artist.name]
)
title_parts = []
title_parts.append(self.track.title)
if self.track.album:
title_parts.append(self.track.album.title)
title_parts.append(self.track.artist.name)
title = " - ".join(title_parts)
filename = "{}.{}".format(title, extension)
tmp_file = tempfile.TemporaryFile()
for chunk in r.iter_content(chunk_size=512):
@ -1126,7 +1130,7 @@ class LibraryQuerySet(models.QuerySet):
)
def viewable_by(self, actor):
from funkwhale_api.federation.models import LibraryFollow
from funkwhale_api.federation.models import LibraryFollow, Follow
if actor is None:
return self.filter(privacy_level="everyone")
@ -1136,11 +1140,17 @@ class LibraryQuerySet(models.QuerySet):
followed_libraries = LibraryFollow.objects.filter(
actor=actor, approved=True
).values_list("target", flat=True)
followed_channels_libraries = (
Follow.objects.exclude(target__channel=None)
.filter(actor=actor, approved=True,)
.values_list("target__channel__library", flat=True)
)
return self.filter(
me_query
| instance_query
| models.Q(privacy_level="everyone")
| models.Q(pk__in=followed_libraries)
| models.Q(pk__in=followed_channels_libraries)
)
@ -1174,7 +1184,7 @@ class Library(federation_models.FederationMixin):
return "/library/{}".format(self.uuid)
def save(self, **kwargs):
if not self.pk and not self.fid and self.actor.get_user():
if not self.pk and not self.fid and self.actor.is_local:
self.fid = self.get_federation_id()
self.followers_url = self.fid + "/followers"
@ -1266,7 +1276,11 @@ class TrackActor(models.Model):
).values_list("id", "track")
objs = []
if library.privacy_level == "me":
follow_queryset = library.received_follows.filter(approved=True).exclude(
if library.get_channel():
follow_queryset = library.channel.actor.received_follows
else:
follow_queryset = library.received_follows
follow_queryset = follow_queryset.filter(approved=True).exclude(
actor__user__isnull=True
)
if actor_ids:

1
api/requirements/base.txt
Wyświetl plik

@ -79,3 +79,4 @@ click>=7,<8
service_identity==18.1.0
markdown>=3.2,<4
bleach>=3,<4
feedparser==6.0.0b3

1
api/setup.cfg
Wyświetl plik

@ -31,3 +31,4 @@ env =
FUNKWHALE_SPA_HTML_ROOT=http://noop/
PROXY_MEDIA=true
MUSIC_USE_DENORMALIZATION=true
EXTERNAL_MEDIA_PROXY_ENABLED=true

469
api/tests/audio/test_serializers.py
Wyświetl plik

@ -1,5 +1,7 @@
import datetime
import uuid
import feedparser
import pytest
import pytz
@ -8,6 +10,7 @@ from django.templatetags.static import static
from funkwhale_api.audio import serializers
from funkwhale_api.common import serializers as common_serializers
from funkwhale_api.common import utils as common_utils
from funkwhale_api.federation import actors
from funkwhale_api.federation import serializers as federation_serializers
from funkwhale_api.federation import utils as federation_utils
from funkwhale_api.music import serializers as music_serializers
@ -232,6 +235,28 @@ def test_channel_serializer_representation(factories, to_api_date):
assert serializers.ChannelSerializer(channel).data == expected
def test_channel_serializer_external_representation(factories, to_api_date):
content = factories["common.Content"]()
channel = factories["audio.Channel"](artist__description=content, external=True)
expected = {
"artist": music_serializers.serialize_artist_simple(channel.artist),
"uuid": str(channel.uuid),
"creation_date": to_api_date(channel.creation_date),
"actor": None,
"attributed_to": federation_serializers.APIActorSerializer(
channel.attributed_to
).data,
"metadata": {},
"rss_url": channel.get_rss_url(),
}
expected["artist"]["description"] = common_serializers.ContentSerializer(
content
).data
assert serializers.ChannelSerializer(channel).data == expected
def test_channel_serializer_representation_subscriptions_count(factories, to_api_date):
channel = factories["audio.Channel"]()
factories["federation.Follow"](target=channel.actor)
@ -351,7 +376,12 @@ def test_rss_channel_serializer(factories):
"href": channel.get_rss_url(),
"rel": "self",
"type": "application/rss+xml",
}
},
{
"href": channel.actor.fid,
"rel": "alternate",
"type": "application/activity+json",
},
],
}
@ -446,3 +476,440 @@ def test_channel_metadata_serializer_validation():
payload.pop("unknown_key")
assert serializer.validated_data == payload
def test_rss_feed_serializer_create(db, now):
rss_url = "http://example.rss/"
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>http://public.url</link>
<atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/>
<lastBuildDate>Wed, 11 Mar 2020 16:01:08 GMT</lastBuildDate>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<ttl>30</ttl>
<language>en</language>
<copyright>2019 Tests</copyright>
<itunes:keywords>pop rock</itunes:keywords>
<image>
<url>
https://image.url
</url>
<title>Image caption</title>
</image>
<itunes:image href="https://image.url"/>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:type>episodic</itunes:type>
<itunes:author>Author</itunes:author>
<itunes:summary><![CDATA[Some content]]></itunes:summary>
<itunes:owner>
<itunes:name>Name</itunes:name>
<itunes:email>email@domain</itunes:email>
</itunes:owner>
<itunes:explicit>yes</itunes:explicit>
<itunes:keywords/>
<itunes:category text="Business">
<itunes:category text="Entrepreneurship">
</itunes:category>
</channel>
</rss>
"""
parsed_feed = feedparser.parse(xml_payload)
serializer = serializers.RssFeedSerializer(data=parsed_feed.feed)
assert serializer.is_valid(raise_exception=True) is True
channel = serializer.save(rss_url)
assert channel.rss_url == "http://real.rss.url"
assert channel.attributed_to == actors.get_service_actor()
assert channel.library.actor == actors.get_service_actor()
assert channel.artist.name == "Hello"
assert channel.artist.attributed_to == actors.get_service_actor()
assert channel.artist.description.content_type == "text/plain"
assert channel.artist.description.text == "Some content"
assert channel.artist.attachment_cover.url == "https://image.url"
assert channel.artist.get_tags() == ["pop", "rock"]
assert channel.actor.url == "http://public.url"
assert channel.actor.last_fetch_date == now
assert channel.metadata == {
"explicit": True,
"copyright": "2019 Tests",
"owner_name": "Name",
"owner_email": "email@domain",
"itunes_category": "Business",
"itunes_subcategory": "Entrepreneurship",
"language": "en",
}
def test_rss_feed_serializer_update(factories, now):
rss_url = "http://example.rss/"
channel = factories["audio.Channel"](rss_url=rss_url, external=True)
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>http://public.url</link>
<atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/>
<lastBuildDate>Wed, 11 Mar 2020 16:01:08 GMT</lastBuildDate>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<ttl>30</ttl>
<language>en</language>
<copyright>2019 Tests</copyright>
<itunes:keywords>pop rock</itunes:keywords>
<image>
<url>
https://image.url
</url>
<title>Image caption</title>
</image>
<itunes:image href="https://image.url"/>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:type>episodic</itunes:type>
<itunes:author>Author</itunes:author>
<itunes:summary><![CDATA[Some content]]></itunes:summary>
<itunes:owner>
<itunes:name>Name</itunes:name>
<itunes:email>email@domain</itunes:email>
</itunes:owner>
<itunes:explicit>yes</itunes:explicit>
<itunes:keywords/>
<itunes:category text="Business">
<itunes:category text="Entrepreneurship">
</itunes:category>
</channel>
</rss>
"""
parsed_feed = feedparser.parse(xml_payload)
serializer = serializers.RssFeedSerializer(data=parsed_feed.feed)
assert serializer.is_valid(raise_exception=True) is True
serializer.save(rss_url)
channel.refresh_from_db()
assert channel.rss_url == "http://real.rss.url"
assert channel.attributed_to == actors.get_service_actor()
assert channel.library.actor == actors.get_service_actor()
assert channel.library.fid is not None
assert channel.artist.name == "Hello"
assert channel.artist.attributed_to == actors.get_service_actor()
assert channel.artist.description.content_type == "text/plain"
assert channel.artist.description.text == "Some content"
assert channel.artist.attachment_cover.url == "https://image.url"
assert channel.artist.get_tags() == ["pop", "rock"]
assert channel.actor.url == "http://public.url"
assert channel.actor.last_fetch_date == now
assert channel.metadata == {
"explicit": True,
"copyright": "2019 Tests",
"owner_name": "Name",
"owner_email": "email@domain",
"itunes_category": "Business",
"itunes_subcategory": "Entrepreneurship",
"language": "en",
}
def test_rss_feed_item_serializer_create(factories):
rss_url = "http://example.rss/"
channel = factories["audio.Channel"](rss_url=rss_url, external=True)
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>http://public.url</link>
<atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/>
<item>
<title>Episode 33</title>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:summary><![CDATA[<p>Html content</p>]]></itunes:summary>
<guid isPermaLink="false"><![CDATA[16f66fff-41ae-4a1c-9101-2746218c4f32]]></guid>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<itunes:duration>00:22:37</itunes:duration>
<itunes:keywords>pop rock</itunes:keywords>
<itunes:season>2</itunes:season>
<itunes:episode>33</itunes:episode>
<itunes:image href="https://image.url/" />
<description><![CDATA[Html content]]></description>
<link>http://public.url/</link>
<enclosure url="https://file.domain/audio.mp3" length="54315884" type="audio/mpeg"/>
</item>
</channel>
</rss>
"""
parsed_feed = feedparser.parse(xml_payload)
entry = parsed_feed.entries[0]
serializer = serializers.RssFeedItemSerializer(data=entry)
assert serializer.is_valid(raise_exception=True) is True
upload = serializer.save(channel, copyright="test something")
expected_uuid = uuid.uuid3(
uuid.NAMESPACE_URL,
"rss://{}-16f66fff-41ae-4a1c-9101-2746218c4f32".format(channel.pk),
)
assert upload.library == channel.library
assert upload.import_status == "finished"
assert upload.source == "https://file.domain/audio.mp3"
assert upload.size == 54315884
assert upload.duration == 1357
assert upload.mimetype == "audio/mpeg"
assert upload.track.uuid == expected_uuid
assert upload.track.artist == channel.artist
assert upload.track.copyright == "test something"
assert upload.track.position == 33
assert upload.track.disc_number == 2
assert upload.track.creation_date == datetime.datetime(2020, 3, 11, 16).replace(
tzinfo=pytz.utc
)
assert upload.track.get_tags() == ["pop", "rock"]
assert upload.track.attachment_cover.url == "https://image.url/"
assert upload.track.description.text == "<p>Html content</p>"
assert upload.track.description.content_type == "text/html"
def test_rss_feed_item_serializer_update(factories):
rss_url = "http://example.rss/"
channel = factories["audio.Channel"](rss_url=rss_url, external=True)
expected_uuid = uuid.uuid3(
uuid.NAMESPACE_URL,
"rss://{}-16f66fff-41ae-4a1c-9101-2746218c4f32".format(channel.pk),
)
upload = factories["music.Upload"](
track__uuid=expected_uuid,
source="https://file.domain/audio.mp3",
library=channel.library,
track__artist=channel.artist,
)
track = upload.track
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>http://public.url</link>
<atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/>
<item>
<title>Episode 33</title>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:summary><![CDATA[<p>Html content</p>]]></itunes:summary>
<guid isPermaLink="false"><![CDATA[16f66fff-41ae-4a1c-9101-2746218c4f32]]></guid>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<itunes:duration>00:22:37</itunes:duration>
<itunes:keywords>pop rock</itunes:keywords>
<itunes:season>2</itunes:season>
<itunes:episode>33</itunes:episode>
<itunes:image href="https://image.url/" />
<description><![CDATA[Html content]]></description>
<link>http://public.url/</link>
<enclosure url="https://file.domain/audio.mp3" length="54315884" type="audio/mpeg"/>
</item>
</channel>
</rss>
"""
parsed_feed = feedparser.parse(xml_payload)
entry = parsed_feed.entries[0]
serializer = serializers.RssFeedItemSerializer(data=entry)
assert serializer.is_valid(raise_exception=True) is True
serializer.save(channel, copyright="test something")
upload.refresh_from_db()
assert upload.track == track
assert upload.library == channel.library
assert upload.import_status == "finished"
assert upload.source == "https://file.domain/audio.mp3"
assert upload.size == 54315884
assert upload.duration == 1357
assert upload.mimetype == "audio/mpeg"
assert upload.track.uuid == expected_uuid
assert upload.track.artist == channel.artist
assert upload.track.copyright == "test something"
assert upload.track.position == 33
assert upload.track.disc_number == 2
assert upload.track.creation_date == datetime.datetime(2020, 3, 11, 16).replace(
tzinfo=pytz.utc
)
assert upload.track.get_tags() == ["pop", "rock"]
assert upload.track.attachment_cover.url == "https://image.url/"
assert upload.track.description.text == "<p>Html content</p>"
assert upload.track.description.content_type == "text/html"
def test_get_channel_from_rss_url(db, r_mock, mocker):
rss_url = "http://example.rss/"
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>http://public.url</link>
<atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/>
<lastBuildDate>Wed, 11 Mar 2020 16:01:08 GMT</lastBuildDate>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<ttl>30</ttl>
<language>en</language>
<copyright>2019 Tests</copyright>
<itunes:keywords>pop rock</itunes:keywords>
<image>
<url>
https://image.url
</url>
<title>Image caption</title>
</image>
<itunes:image href="https://image.url"/>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:type>episodic</itunes:type>
<itunes:author>Author</itunes:author>
<itunes:summary><![CDATA[Some content]]></itunes:summary>
<itunes:owner>
<itunes:name>Name</itunes:name>
<itunes:email>email@domain</itunes:email>
</itunes:owner>
<itunes:explicit>yes</itunes:explicit>
<itunes:keywords/>
<itunes:category text="Business">
<itunes:category text="Entrepreneurship">
</itunes:category>
<item>
<title>Episode 33</title>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:summary><![CDATA[<p>Html content</p>]]></itunes:summary>
<guid isPermaLink="false"><![CDATA[16f66fff-41ae-4a1c-9101-2746218c4f32]]></guid>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<itunes:duration>00:22:37</itunes:duration>
<itunes:keywords>pop rock</itunes:keywords>
<itunes:season>2</itunes:season>
<itunes:episode>33</itunes:episode>
<itunes:image href="https://image.url/" />
<description><![CDATA[Html content]]></description>
<link>http://public.url/</link>
<enclosure url="https://file.domain/audio.mp3" length="54315884" type="audio/mpeg"/>
</item>
<item>
<title>Episode 32</title>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:summary><![CDATA[<p>Html content</p>]]></itunes:summary>
<guid isPermaLink="false"><![CDATA[16f66fff-41ae-4a1c-910e-2746218c4f32]]></guid>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<itunes:duration>00:22:37</itunes:duration>
<itunes:keywords>pop rock</itunes:keywords>
<itunes:season>2</itunes:season>
<itunes:episode>32</itunes:episode>
<itunes:image href="https://image.url/" />
<description><![CDATA[Html content]]></description>
<link>http://public.url/</link>
<enclosure url="https://file.domain/audio2.mp3" length="54315884" type="audio/mpeg"/>
</item>
<item>
<title>Ignored, missing enĉlosure</title>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:summary><![CDATA[<p>Html content</p>]]></itunes:summary>
<guid isPermaLink="false"><![CDATA[16f66fff-41ae-4a1c-910e-2746218c4f32]]></guid>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<itunes:duration>00:22:37</itunes:duration>
<itunes:keywords>pop rock</itunes:keywords>
<itunes:season>2</itunes:season>
<itunes:episode>32</itunes:episode>
<itunes:image href="https://image.url/" />
<description><![CDATA[Html content]]></description>
<link>http://public.url/</link>
</item>
</channel>
</rss>
"""
parsed_feed = feedparser.parse(xml_payload)
r_mock.get(rss_url, text=xml_payload)
feed_init = mocker.spy(serializers.RssFeedSerializer, "__init__")
feed_save = mocker.spy(serializers.RssFeedSerializer, "save")
item_init = mocker.spy(serializers.RssFeedItemSerializer, "__init__")
item_save = mocker.spy(serializers.RssFeedItemSerializer, "save")
on_commit = mocker.spy(common_utils, "on_commit")
channel, uploads = serializers.get_channel_from_rss_url(rss_url)
assert channel.artist.name == "Hello"
serializer_instance = feed_init.call_args[0][0]
feed_init.assert_called_once_with(serializer_instance, data=parsed_feed.feed)
feed_save.assert_called_once_with(serializer_instance, rss_url)
for i in [0, 1]:
serializer_instance = item_init.call_args_list[i][0][0]
item_init.assert_any_call(serializer_instance, data=parsed_feed.entries[i])
item_save.assert_any_call(
serializer_instance, channel, existing_uploads=[], copyright="2019 Tests"
)
assert len(uploads) == 2
assert channel.library.uploads.count() == 2
on_commit.assert_any_call(
serializers.music_models.TrackActor.create_entries,
library=channel.library,
delete_existing=True,
)
def test_get_channel_from_rss_honor_mrf_inbox_before_http(
mrf_inbox_registry, factories, mocker
):
apply = mocker.patch.object(mrf_inbox_registry, "apply", return_value=(None, False))
rss_url = "https://rss.domain/test"
with pytest.raises(serializers.FeedFetchException, match=r".*blocked.*"):
serializers.get_channel_from_rss_url(rss_url)
apply.assert_any_call({"id": rss_url})
def test_get_channel_from_rss_honor_mrf_inbox_after_http(
mrf_inbox_registry, r_mock, mocker, db
):
apply = mocker.patch.object(
mrf_inbox_registry,
"apply",
side_effect=[(True, False), (True, False), (None, False)],
)
rss_url = "https://rss.domain/test"
# the feed has a redirection, we check both urls
final_rss_url = "https://real.rss.domain/test"
public_url = "http://public.url"
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>{}</link>
<atom:link rel="self" type="application/rss+xml" href="{}"/>
<language>en</language>
<copyright>2019 Tests</copyright>
<itunes:keywords>pop rock</itunes:keywords>
</channel>
</rss>
""".format(
public_url, final_rss_url
)
r_mock.get(rss_url, text=xml_payload)
with pytest.raises(serializers.FeedFetchException, match=r".*blocked.*"):
serializers.get_channel_from_rss_url(rss_url)
apply.assert_any_call({"id": rss_url})
apply.assert_any_call({"id": final_rss_url})
apply.assert_any_call({"id": public_url})

2
api/tests/audio/test_spa_views.py
Wyświetl plik

@ -11,7 +11,7 @@ from funkwhale_api.music import serializers
@pytest.mark.parametrize("attribute", ["uuid", "actor.full_username"])
def test_channel_detail(attribute, spa_html, no_api_auth, client, factories, settings):
channel = factories["audio.Channel"]()
channel = factories["audio.Channel"](library__privacy_level="everyone")
factories["music.Upload"](playable=True, library=channel.library)
url = "/channels/{}".format(utils.recursive_getattr(channel, attribute))
detail_url = "/channels/{}".format(channel.actor.full_username)

53
api/tests/audio/test_tasks.py 100644
Wyświetl plik

@ -0,0 +1,53 @@
import datetime
import pytest
from funkwhale_api.audio import tasks
def test_fetch_rss_feeds(factories, settings, now, mocker):
settings.PODCASTS_RSS_FEED_REFRESH_DELAY = 5
prunable_date = now - datetime.timedelta(
seconds=settings.PODCASTS_RSS_FEED_REFRESH_DELAY
)
fetch_rss_feed = mocker.patch.object(tasks.fetch_rss_feed, "delay")
channels = [
# recent, not fetched
factories["audio.Channel"](actor__last_fetch_date=now, external=True),
# old but not external, not fetched
factories["audio.Channel"](actor__last_fetch_date=prunable_date),
# old and external, fetched !
factories["audio.Channel"](actor__last_fetch_date=prunable_date, external=True),
factories["audio.Channel"](actor__last_fetch_date=prunable_date, external=True),
]
tasks.fetch_rss_feeds()
assert fetch_rss_feed.call_count == 2
fetch_rss_feed.assert_any_call(rss_url=channels[2].rss_url)
fetch_rss_feed.assert_any_call(rss_url=channels[3].rss_url)
def test_fetch_rss_feed(factories, mocker):
channel = factories["audio.Channel"](external=True)
get_channel_from_rss_url = mocker.patch.object(
tasks.serializers, "get_channel_from_rss_url"
)
tasks.fetch_rss_feed(channel.rss_url)
get_channel_from_rss_url.assert_called_once_with(channel.rss_url)
def test_fetch_rss_feed_blocked_is_deleted(factories, mocker):
channel = factories["audio.Channel"](external=True)
mocker.patch.object(
tasks.serializers,
"get_channel_from_rss_url",
side_effect=tasks.serializers.BlockedFeedException(),
)
tasks.fetch_rss_feed(channel.rss_url)
with pytest.raises(channel.DoesNotExist):
channel.refresh_from_db()

75
api/tests/audio/test_views.py
Wyświetl plik

@ -251,6 +251,19 @@ def test_channel_rss_feed(factories, api_client, preferences):
assert response["Content-Type"] == "application/rss+xml"
def test_channel_rss_feed_redirects_for_external(factories, api_client, preferences):
preferences["common__api_authentication_required"] = False
channel = factories["audio.Channel"](external=True)
factories["music.Upload"](library=channel.library, playable=True)
url = reverse("api:v1:channels-rss", kwargs={"composite": channel.uuid})
response = api_client.get(url)
assert response.status_code == 302
assert response["Location"] == channel.rss_url
def test_channel_rss_feed_remote(factories, api_client, preferences):
preferences["common__api_authentication_required"] = False
channel = factories["audio.Channel"]()
@ -291,3 +304,65 @@ def test_channel_metadata_choices(factories, api_client):
assert response.status_code == 200
assert response.data == expected
def test_subscribe_to_rss_feed_existing_channel(
factories, logged_in_api_client, mocker
):
actor = logged_in_api_client.user.create_actor()
rss_url = "http://example.test/rss.url"
channel = factories["audio.Channel"](rss_url=rss_url, external=True)
url = reverse("api:v1:channels-rss_subscribe")
response = logged_in_api_client.post(url, {"url": rss_url})
assert response.status_code == 201
subscription = actor.emitted_follows.select_related(
"target__channel__artist__description",
"target__channel__artist__attachment_cover",
).latest("id")
assert subscription.target == channel.actor
assert subscription.approved is True
assert subscription.fid == subscription.get_federation_id()
setattr(subscription.target.channel.artist, "_tracks_count", 0)
setattr(subscription.target.channel.artist, "_prefetched_tagged_items", [])
expected = serializers.SubscriptionSerializer(subscription).data
assert response.data == expected
def test_subscribe_to_rss_feed_existing_subscription(
factories, logged_in_api_client, mocker
):
actor = logged_in_api_client.user.create_actor()
rss_url = "http://example.test/rss.url"
channel = factories["audio.Channel"](rss_url=rss_url, external=True)
factories["federation.Follow"](target=channel.actor, approved=True, actor=actor)
url = reverse("api:v1:channels-rss_subscribe")
response = logged_in_api_client.post(url, {"url": rss_url})
assert response.status_code == 201
assert channel.actor.received_follows.count() == 1
def test_subscribe_to_rss_creates_channel(factories, logged_in_api_client, mocker):
logged_in_api_client.user.create_actor()
rss_url = "http://example.test/rss.url"
channel = factories["audio.Channel"]()
get_channel_from_rss_url = mocker.patch.object(
serializers, "get_channel_from_rss_url", return_value=(channel, [])
)
url = reverse("api:v1:channels-rss_subscribe")
response = logged_in_api_client.post(url, {"url": rss_url})
assert response.status_code == 201
assert response.data["channel"]["uuid"] == channel.uuid
get_channel_from_rss_url.assert_called_once_with(rss_url)

2
api/tests/conftest.py
Wyświetl plik

@ -87,6 +87,8 @@ def cache():
"""
yield django_cache
django_cache.clear()
if "service_actor" in actors._CACHE:
del actors._CACHE["service_actor"]
@pytest.fixture(autouse=True)

2
api/tests/federation/test_serializers.py
Wyświetl plik

@ -1444,7 +1444,7 @@ def test_channel_actor_outbox_serializer(factories):
def test_channel_upload_serializer(factories):
channel = factories["audio.Channel"]()
channel = factories["audio.Channel"](library__privacy_level="everyone")
content = factories["common.Content"]()
upload = factories["music.Upload"](
playable=True,

12
api/tests/federation/test_views.py
Wyświetl plik

@ -186,6 +186,18 @@ def test_music_library_retrieve_excludes_channel_libraries(factories, api_client
assert response.status_code == 404
def test_actor_retrieve_excludes_channel_with_private_library(factories, api_client):
channel = factories["audio.Channel"](external=True, library__privacy_level="me")
url = reverse(
"federation:actors-detail",
kwargs={"preferred_username": channel.actor.preferred_username},
)
response = api_client.get(url)
assert response.status_code == 404
def test_music_library_retrieve_page_public(factories, api_client):
library = factories["music.Library"](privacy_level="everyone", actor__local=True)
upload = factories["music.Upload"](library=library, import_status="finished")

4
api/tests/music/test_filters.py
Wyświetl plik

@ -113,7 +113,7 @@ def test_track_filter_tag_multiple(
def test_channel_filter_track(factories, queryset_equal_list, mocker, anonymous_user):
channel = factories["audio.Channel"]()
channel = factories["audio.Channel"](library__privacy_level="everyone")
upload = factories["music.Upload"](
library=channel.library, playable=True, track__artist=channel.artist
)
@ -129,7 +129,7 @@ def test_channel_filter_track(factories, queryset_equal_list, mocker, anonymous_
def test_channel_filter_album(factories, queryset_equal_list, mocker, anonymous_user):
channel = factories["audio.Channel"]()
channel = factories["audio.Channel"](library__privacy_level="everyone")
upload = factories["music.Upload"](
library=channel.library, playable=True, track__artist=channel.artist
)

24
front/src/components/audio/SearchBar.vue
Wyświetl plik

@ -57,7 +57,6 @@ export default {
},
onSelect (result, response) {
jQuery(self.$el).search("set value", searchQuery)
console.log('SELECTEING', result)
router.push(result.routerUrl)
jQuery(self.$el).search("hide results")
return false
@ -83,6 +82,10 @@ export default {
code: 'federation',
name: self.$pgettext('*/*/*', 'Federation'),
},
{
code: 'podcasts',
name: self.$pgettext('*/*/*', 'Podcasts'),
},
{
code: 'artists',
route: 'library.artists.detail',
@ -168,6 +171,25 @@ export default {
}
}
}
else if (category.code === 'podcasts') {
if (objId) {
isEmptyResults = false
let searchMessage = self.$pgettext('Search/*/*', 'Subscribe to podcast via RSS')
results['podcasts'] = {
name: self.$pgettext('*/*/*', 'Podcasts'),
results: [{
title: searchMessage,
routerUrl: {
name: 'search',
query: {
id: objId,
type: "rss"
}
}
}]
}
}
}
else {
initialResponse[category.code].forEach(result => {
isEmptyResults = false

3
front/src/router/index.js
Wyświetl plik

@ -80,7 +80,8 @@ export default new Router({
component: () =>
import(/* webpackChunkName: "core" */ "@/views/Search"),
props: route => ({
initialId: route.query.id
initialId: route.query.id,
type: route.query.type,
})
},
{

154
front/src/views/Search.vue
Wyświetl plik

@ -2,9 +2,12 @@
<main class="main pusher" v-title="labels.title">
<section class="ui vertical stripe segment">
<div class="ui small text container">
<form :class="['ui', {loading: isLoading}, 'form']" @submit.stop.prevent="createFetch">
<h2><translate translate-context="Content/Fetch/Title">Retrieve a remote object</translate></h2>
<p>
<form :class="['ui', {loading: isLoading}, 'form']" @submit.stop.prevent="submit">
<h2>{{ labels.title }}</h2>
<p v-if="type === 'rss'">
<translate translate-context="Content/Fetch/Paragraph">Use this form to subscribe to a podcast using its RSS feed.</translate>
</p>
<p v-else>
<translate translate-context="Content/Fetch/Paragraph">Use this form to retrieve an object hosted somewhere else in the fediverse.</translate>
</p>
<div v-if="errors.length > 0" class="ui negative message">
@ -23,16 +26,9 @@
<translate translate-context="Content/Search/Input.Label/Noun">Search</translate>
</button>
</form>
<div v-if="!isLoading && fetch && fetch.status === 'finished'">
<div class="ui hidden divider"></div>
<h2><translate translate-context="Content/Fetch/Title/Noun">Result</translate></h2>
<div class="ui hidden divider"></div>
<div v-if="objComponent" class="ui app-cards cards">
<component v-bind="objComponent.props" :is="objComponent.type"></component>
</div>
<div v-else class="ui warning message">
<p><translate translate-context="Content/*/Error message.Title">This kind of object isn't supported yet</translate></p>
</div>
<div v-if="!isLoading && fetch && fetch.status === 'finished' && !redirectRoute" class="ui warning message">
<p><translate translate-context="Content/*/Error message.Title">This kind of object isn't supported yet</translate></p>
</div>
</div>
</div>
</section>
@ -43,22 +39,12 @@
import axios from 'axios'
import AlbumCard from '@/components/audio/album/Card'
import ArtistCard from '@/components/audio/artist/Card'
import LibraryCard from '@/views/content/remote/Card'
import ChannelEntryCard from '@/components/audio/ChannelEntryCard'
export default {
props: {
initialId: { type: String, required: false}
},
components: {
ActorLink: () => import(/* webpackChunkName: "common" */ "@/components/common/ActorLink"),
ArtistCard,
AlbumCard,
LibraryCard,
ChannelEntryCard,
initialId: { type: String, required: false},
type: { type: String, required: false},
},
components: {},
data () {
return {
id: this.initialId,
@ -70,14 +56,25 @@ export default {
},
created () {
if (this.id) {
this.createFetch()
if (this.type === 'rss') {
this.rssSubscribe()
} else {
this.createFetch()
}
}
},
computed: {
labels() {
let title = this.$pgettext('Head/Fetch/Title', "Search a remote object")
let fieldLabel = this.$pgettext('Head/Fetch/Field.Placeholder', "URL or @username")
if (this.type === "rss") {
title = this.$pgettext('Head/Fetch/Title', "Subscribe to a podcast RSS feed")
fieldLabel = this.$pgettext('*/*/*', "RSS Feed URL")
}
return {
title: this.$pgettext('Head/Fetch/Title', "Search a remote object"),
fieldLabel: this.$pgettext('Head/Fetch/Field.Placeholder', "URL or @username"),
title,
fieldLabel
}
},
objInfo () {
@ -85,43 +82,38 @@ export default {
return this.fetch.object
}
},
objComponent () {
if (!this.obj) {
redirectRoute () {
if (!this.objInfo) {
return
}
switch (this.objInfo.type) {
case "account":
return {
type: "actor-link",
props: {actor: this.obj}
}
case "library":
return {
type: "library-card",
props: {library: this.obj}
}
case "album":
return {
type: "album-card",
props: {album: this.obj}
}
case "artist":
return {
type: "artist-card",
props: {artist: this.obj}
}
case "upload":
return {
type: "channel-entry-card",
props: {entry: this.obj.track}
}
case 'account':
let [username, domain] = this.objInfo.full_username.split('@')
return {name: 'profile.full', params: {username, domain}}
case 'library':
return {name: 'library.detail', params: {id: this.objInfo.uuid}}
case 'artist':
return {name: 'library.artists.detail', params: {id: this.objInfo.id}}
case 'album':
return {name: 'library.albums.detail', params: {id: this.objInfo.id}}
case 'track':
return {name: 'library.tracks.detail', params: {id: this.objInfo.id}}
case 'upload':
return {name: 'library.uploads.detail', params: {id: this.objInfo.uuid}}
default:
return
break;
}
}
},
methods: {
submit () {
if (this.type === 'rss') {
return this.rssSubscribe()
} else {
return this.createFetch()
}
},
createFetch () {
if (!this.id) {
return
@ -148,58 +140,38 @@ export default {
self.errors = error.backendErrors
})
},
getObj (objInfo) {
rssSubscribe () {
if (!this.id) {
return
}
this.$router.replace({name: "search", query: {id: this.id, type: 'rss'}})
this.fetch = null
let self = this
self.errors = []
self.isLoading = true
let url = null
switch (objInfo.type) {
case 'account':
url = `federation/actors/${objInfo.full_username}/`
break;
case 'library':
url = `libraries/${objInfo.uuid}/`
break;
case 'artist':
url = `artists/${objInfo.id}/`
break;
case 'album':
url = `albums/${objInfo.id}/`
break;
case 'upload':
url = `uploads/${objInfo.uuid}/`
break;
let payload = {
url: this.id
}
default:
break;
}
if (!url) {
this.errors.push(
self.$pgettext("Content/*/Error message.Title", "This kind of object isn't supported yet")
)
this.isLoading = false
return
}
axios.get(url).then((response) => {
self.obj = response.data
axios.post('channels/rss-subscribe/', payload).then((response) => {
self.isLoading = false
self.$store.commit('channels/subscriptions', {uuid: response.data.channel.uuid, value: true})
self.$router.push({name: 'channels.detail', params: {id: response.data.channel.uuid}})
}, error => {
self.isLoading = false
self.errors = error.backendErrors
})
}
},
},
watch: {
initialId (v) {
this.id = v
this.createFetch()
},
objInfo (v) {
this.obj = null
redirectRoute (v) {
if (v) {
this.getObj(v)
this.$router.push(v)
}
}
}

6
front/src/views/channels/DetailBase.vue
Wyświetl plik

@ -14,7 +14,7 @@
</div>
<div class="ui column right aligned">
<tags-list v-if="object.artist.tags && object.artist.tags.length > 0" :tags="object.artist.tags"></tags-list>
<actor-link :avatar="false" :actor="object.attributed_to" :display-name="true"></actor-link>
<actor-link v-if="object.actor" :avatar="false" :actor="object.attributed_to" :display-name="true"></actor-link>
<template v-if="totalTracks > 0">
<div class="ui hidden very small divider"></div>
<translate translate-context="Content/Channel/Paragraph"
@ -125,7 +125,7 @@
<div class="left aligned" :title="object.artist.name">
{{ object.artist.name }}
<div class="ui hidden very small divider"></div>
<div class="sub header ellipsis" :title="object.actor.full_username">
<div class="sub header ellipsis" v-if="object.actor ":title="object.actor.full_username">
{{ object.actor.full_username }}
</div>
</div>
@ -268,7 +268,7 @@ export default {
this.isLoading = true
let channelPromise = axios.get(`channels/${this.id}`).then(response => {
self.object = response.data
if (self.id == response.data.uuid && response.data.actor) {
if ((self.id == response.data.uuid) && response.data.actor) {
// replace with the pretty channel url if possible
let actor = response.data.actor
if (actor.is_local) {