diff --git a/api/funkwhale_api/common/serializers.py b/api/funkwhale_api/common/serializers.py
new file mode 100644
index 000000000..62d9c567e
--- /dev/null
+++ b/api/funkwhale_api/common/serializers.py
@@ -0,0 +1,76 @@
+from rest_framework import serializers
+
+
+class ActionSerializer(serializers.Serializer):
+ """
+ A special serializer that can operate on a list of objects
+ and apply actions on it.
+ """
+
+ action = serializers.CharField(required=True)
+ objects = serializers.JSONField(required=True)
+ filters = serializers.DictField(required=False)
+ actions = None
+ filterset_class = None
+
+ def __init__(self, *args, **kwargs):
+ self.queryset = kwargs.pop('queryset')
+ if self.actions is None:
+ raise ValueError(
+ 'You must declare a list of actions on '
+ 'the serializer class')
+
+ for action in self.actions:
+ handler_name = 'handle_{}'.format(action)
+ assert hasattr(self, handler_name), (
+ '{} miss a {} method'.format(
+ self.__class__.__name__, handler_name)
+ )
+ super().__init__(self, *args, **kwargs)
+
+ def validate_action(self, value):
+ if value not in self.actions:
+ raise serializers.ValidationError(
+ '{} is not a valid action. Pick one of {}.'.format(
+ value, ', '.join(self.actions)
+ )
+ )
+ return value
+
+ def validate_objects(self, value):
+ qs = None
+ if value == 'all':
+ return self.queryset.all().order_by('id')
+ if type(value) in [list, tuple]:
+ return self.queryset.filter(pk__in=value).order_by('id')
+
+ raise serializers.ValidationError(
+ '{} is not a valid value for objects. You must provide either a '
+ 'list of identifiers or the string "all".'.format(value))
+
+ def validate(self, data):
+ if self.filterset_class and 'filters' in data:
+ qs_filterset = self.filterset_class(
+ data['filters'], queryset=data['objects'])
+ try:
+ assert qs_filterset.form.is_valid()
+ except (AssertionError, TypeError):
+ raise serializers.ValidationError('Invalid filters')
+ data['objects'] = qs_filterset.qs
+
+ data['count'] = data['objects'].count()
+ if data['count'] < 1:
+ raise serializers.ValidationError(
+ 'No object matching your request')
+ return data
+
+ def save(self):
+ handler_name = 'handle_{}'.format(self.validated_data['action'])
+ handler = getattr(self, handler_name)
+ result = handler(self.validated_data['objects'])
+ payload = {
+ 'updated': self.validated_data['count'],
+ 'action': self.validated_data['action'],
+ 'result': result,
+ }
+ return payload
diff --git a/api/funkwhale_api/federation/filters.py b/api/funkwhale_api/federation/filters.py
index 7a388ff12..1d93f68b9 100644
--- a/api/funkwhale_api/federation/filters.py
+++ b/api/funkwhale_api/federation/filters.py
@@ -24,7 +24,7 @@ class LibraryFilter(django_filters.FilterSet):
class LibraryTrackFilter(django_filters.FilterSet):
library = django_filters.CharFilter('library__uuid')
- imported = django_filters.CharFilter(method='filter_imported')
+ status = django_filters.CharFilter(method='filter_status')
q = fields.SearchFilter(search_fields=[
'artist_name',
'title',
@@ -32,11 +32,15 @@ class LibraryTrackFilter(django_filters.FilterSet):
'library__actor__domain',
])
- def filter_imported(self, queryset, field_name, value):
- if value.lower() in ['true', '1', 'yes']:
- queryset = queryset.filter(local_track_file__isnull=False)
- elif value.lower() in ['false', '0', 'no']:
- queryset = queryset.filter(local_track_file__isnull=True)
+ def filter_status(self, queryset, field_name, value):
+ if value == 'imported':
+ return queryset.filter(local_track_file__isnull=False)
+ elif value == 'not_imported':
+ return queryset.filter(
+ local_track_file__isnull=True
+ ).exclude(import_jobs__status='pending')
+ elif value == 'import_pending':
+ return queryset.filter(import_jobs__status='pending')
return queryset
class Meta:
diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py
index 51561e222..6ffffaa9a 100644
--- a/api/funkwhale_api/federation/serializers.py
+++ b/api/funkwhale_api/federation/serializers.py
@@ -10,8 +10,11 @@ from rest_framework import serializers
from dynamic_preferences.registries import global_preferences_registry
from funkwhale_api.common import utils as funkwhale_utils
-
+from funkwhale_api.common import serializers as common_serializers
+from funkwhale_api.music import models as music_models
+from funkwhale_api.music import tasks as music_tasks
from . import activity
+from . import filters
from . import models
from . import utils
@@ -293,6 +296,7 @@ class APILibraryCreateSerializer(serializers.ModelSerializer):
class APILibraryTrackSerializer(serializers.ModelSerializer):
library = APILibrarySerializer()
+ status = serializers.SerializerMethodField()
class Meta:
model = models.LibraryTrack
@@ -311,8 +315,20 @@ class APILibraryTrackSerializer(serializers.ModelSerializer):
'title',
'library',
'local_track_file',
+ 'status',
]
+ def get_status(self, o):
+ try:
+ if o.local_track_file is not None:
+ return 'imported'
+ except music_models.TrackFile.DoesNotExist:
+ pass
+ for job in o.import_jobs.all():
+ if job.status == 'pending':
+ return 'import_pending'
+ return 'not_imported'
+
class FollowSerializer(serializers.Serializer):
id = serializers.URLField(max_length=500)
@@ -806,3 +822,29 @@ class CollectionSerializer(serializers.Serializer):
if self.context.get('include_ap_context', True):
d['@context'] = AP_CONTEXT
return d
+
+
+class LibraryTrackActionSerializer(common_serializers.ActionSerializer):
+ actions = ['import']
+ filterset_class = filters.LibraryTrackFilter
+
+ @transaction.atomic
+ def handle_import(self, objects):
+ batch = music_models.ImportBatch.objects.create(
+ source='federation',
+ submitted_by=self.context['submitted_by']
+ )
+ jobs = []
+ for lt in objects:
+ job = music_models.ImportJob(
+ batch=batch,
+ library_track=lt,
+ mbid=lt.mbid,
+ source=lt.url,
+ )
+ jobs.append(job)
+
+ music_models.ImportJob.objects.bulk_create(jobs)
+ music_tasks.import_batch_run.delay(import_batch_id=batch.pk)
+
+ return {'batch': {'id': batch.pk}}
diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py
index 06a2cd040..1350ec731 100644
--- a/api/funkwhale_api/federation/views.py
+++ b/api/funkwhale_api/federation/views.py
@@ -15,7 +15,7 @@ from rest_framework.serializers import ValidationError
from funkwhale_api.common import preferences
from funkwhale_api.common import utils as funkwhale_utils
-from funkwhale_api.music.models import TrackFile
+from funkwhale_api.music import models as music_models
from funkwhale_api.users.permissions import HasUserPermission
from . import activity
@@ -148,7 +148,9 @@ class MusicFilesViewSet(FederationMixin, viewsets.GenericViewSet):
def list(self, request, *args, **kwargs):
page = request.GET.get('page')
library = actors.SYSTEM_ACTORS['library'].get_actor_instance()
- qs = TrackFile.objects.order_by('-creation_date').select_related(
+ qs = music_models.TrackFile.objects.order_by(
+ '-creation_date'
+ ).select_related(
'track__artist',
'track__album__artist'
).filter(library_track__isnull=True)
@@ -294,7 +296,7 @@ class LibraryTrackViewSet(
'library__actor',
'library__follow',
'local_track_file',
- )
+ ).prefetch_related('import_jobs')
filter_class = filters.LibraryTrackFilter
serializer_class = serializers.APILibraryTrackSerializer
ordering_fields = (
@@ -307,3 +309,16 @@ class LibraryTrackViewSet(
'fetched_date',
'published_date',
)
+
+ @list_route(methods=['post'])
+ def action(self, request, *args, **kwargs):
+ queryset = models.LibraryTrack.objects.filter(
+ local_track_file__isnull=True)
+ serializer = serializers.LibraryTrackActionSerializer(
+ request.data,
+ queryset=queryset,
+ context={'submitted_by': request.user}
+ )
+ serializer.is_valid(raise_exception=True)
+ result = serializer.save()
+ return response.Response(result, status=200)
diff --git a/api/funkwhale_api/music/serializers.py b/api/funkwhale_api/music/serializers.py
index c77983a40..b72bb8c4a 100644
--- a/api/funkwhale_api/music/serializers.py
+++ b/api/funkwhale_api/music/serializers.py
@@ -250,28 +250,6 @@ class TrackActivitySerializer(activity_serializers.ModelSerializer):
return 'Audio'
-class SubmitFederationTracksSerializer(serializers.Serializer):
- library_tracks = serializers.PrimaryKeyRelatedField(
- many=True,
- queryset=LibraryTrack.objects.filter(local_track_file__isnull=True),
- )
-
- @transaction.atomic
- def save(self, **kwargs):
- batch = models.ImportBatch.objects.create(
- source='federation',
- **kwargs
- )
- for lt in self.validated_data['library_tracks']:
- models.ImportJob.objects.create(
- batch=batch,
- library_track=lt,
- mbid=lt.mbid,
- source=lt.url,
- )
- return batch
-
-
class ImportJobRunSerializer(serializers.Serializer):
jobs = serializers.PrimaryKeyRelatedField(
many=True,
diff --git a/api/funkwhale_api/music/tasks.py b/api/funkwhale_api/music/tasks.py
index e5426904a..993456c27 100644
--- a/api/funkwhale_api/music/tasks.py
+++ b/api/funkwhale_api/music/tasks.py
@@ -173,6 +173,13 @@ def import_job_run(self, import_job, replace=False, use_acoustid=False):
raise
+@celery.app.task(name='ImportBatch.run')
+@celery.require_instance(models.ImportBatch, 'import_batch')
+def import_batch_run(import_batch):
+ for job_id in import_batch.jobs.order_by('id').values_list('id', flat=True):
+ import_job_run.delay(import_job_id=job_id)
+
+
@celery.app.task(name='Lyrics.fetch_content')
@celery.require_instance(models.Lyrics, 'lyrics')
def fetch_content(lyrics):
diff --git a/api/funkwhale_api/music/views.py b/api/funkwhale_api/music/views.py
index 5e3a7a4c1..aa07ad52c 100644
--- a/api/funkwhale_api/music/views.py
+++ b/api/funkwhale_api/music/views.py
@@ -449,22 +449,6 @@ class SubmitViewSet(viewsets.ViewSet):
data, request, batch=None, import_request=import_request)
return Response(import_data)
- @list_route(methods=['post'])
- @transaction.non_atomic_requests
- def federation(self, request, *args, **kwargs):
- serializer = serializers.SubmitFederationTracksSerializer(
- data=request.data)
- serializer.is_valid(raise_exception=True)
- batch = serializer.save(submitted_by=request.user)
- for job in batch.jobs.all():
- funkwhale_utils.on_commit(
- tasks.import_job_run.delay,
- import_job_id=job.pk,
- use_acoustid=False,
- )
-
- return Response({'id': batch.id}, status=201)
-
@transaction.atomic
def _import_album(self, data, request, batch=None, import_request=None):
# we import the whole album here to prevent race conditions that occurs
diff --git a/api/tests/common/test_serializers.py b/api/tests/common/test_serializers.py
new file mode 100644
index 000000000..563676556
--- /dev/null
+++ b/api/tests/common/test_serializers.py
@@ -0,0 +1,100 @@
+import django_filters
+
+from funkwhale_api.common import serializers
+from funkwhale_api.users import models
+
+
+class TestActionFilterSet(django_filters.FilterSet):
+ class Meta:
+ model = models.User
+ fields = ['is_active']
+
+
+class TestSerializer(serializers.ActionSerializer):
+ actions = ['test']
+ filterset_class = TestActionFilterSet
+
+ def handle_test(self, objects):
+ return {'hello': 'world'}
+
+
+def test_action_serializer_validates_action():
+ data = {'objects': 'all', 'action': 'nope'}
+ serializer = TestSerializer(data, queryset=models.User.objects.none())
+
+ assert serializer.is_valid() is False
+ assert 'action' in serializer.errors
+
+
+def test_action_serializer_validates_objects():
+ data = {'objects': 'nope', 'action': 'test'}
+ serializer = TestSerializer(data, queryset=models.User.objects.none())
+
+ assert serializer.is_valid() is False
+ assert 'objects' in serializer.errors
+
+
+def test_action_serializers_objects_clean_ids(factories):
+ user1 = factories['users.User']()
+ user2 = factories['users.User']()
+
+ data = {'objects': [user1.pk], 'action': 'test'}
+ serializer = TestSerializer(data, queryset=models.User.objects.all())
+
+ assert serializer.is_valid() is True
+ assert list(serializer.validated_data['objects']) == [user1]
+
+
+def test_action_serializers_objects_clean_all(factories):
+ user1 = factories['users.User']()
+ user2 = factories['users.User']()
+
+ data = {'objects': 'all', 'action': 'test'}
+ serializer = TestSerializer(data, queryset=models.User.objects.all())
+
+ assert serializer.is_valid() is True
+ assert list(serializer.validated_data['objects']) == [user1, user2]
+
+
+def test_action_serializers_save(factories, mocker):
+ handler = mocker.spy(TestSerializer, 'handle_test')
+ user1 = factories['users.User']()
+ user2 = factories['users.User']()
+
+ data = {'objects': 'all', 'action': 'test'}
+ serializer = TestSerializer(data, queryset=models.User.objects.all())
+
+ assert serializer.is_valid() is True
+ result = serializer.save()
+ assert result == {
+ 'updated': 2,
+ 'action': 'test',
+ 'result': {'hello': 'world'},
+ }
+ handler.assert_called_once()
+
+
+def test_action_serializers_filterset(factories):
+ user1 = factories['users.User'](is_active=False)
+ user2 = factories['users.User'](is_active=True)
+
+ data = {
+ 'objects': 'all',
+ 'action': 'test',
+ 'filters': {'is_active': True},
+ }
+ serializer = TestSerializer(data, queryset=models.User.objects.all())
+
+ assert serializer.is_valid() is True
+ assert list(serializer.validated_data['objects']) == [user2]
+
+
+def test_action_serializers_validates_at_least_one_object():
+ data = {
+ 'objects': 'all',
+ 'action': 'test',
+ }
+ serializer = TestSerializer(data, queryset=models.User.objects.none())
+
+ assert serializer.is_valid() is False
+ assert 'non_field_errors' in serializer.errors
diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py
index f298c61f5..fcf2ba1b6 100644
--- a/api/tests/federation/test_serializers.py
+++ b/api/tests/federation/test_serializers.py
@@ -699,3 +699,26 @@ def test_api_library_create_serializer_save(factories, r_mock):
assert library.tracks_count == 10
assert library.actor == actor
assert library.follow == follow
+
+
+def test_tapi_library_track_serializer_not_imported(factories):
+ lt = factories['federation.LibraryTrack']()
+ serializer = serializers.APILibraryTrackSerializer(lt)
+
+ assert serializer.get_status(lt) == 'not_imported'
+
+
+def test_tapi_library_track_serializer_imported(factories):
+ tf = factories['music.TrackFile'](federation=True)
+ lt = tf.library_track
+ serializer = serializers.APILibraryTrackSerializer(lt)
+
+ assert serializer.get_status(lt) == 'imported'
+
+
+def test_tapi_library_track_serializer_import_pending(factories):
+ job = factories['music.ImportJob'](federation=True, status='pending')
+ lt = job.library_track
+ serializer = serializers.APILibraryTrackSerializer(lt)
+
+ assert serializer.get_status(lt) == 'import_pending'
diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py
index 10237ed9f..04a419aed 100644
--- a/api/tests/federation/test_views.py
+++ b/api/tests/federation/test_views.py
@@ -418,3 +418,39 @@ def test_can_filter_pending_follows(factories, superuser_api_client):
assert response.status_code == 200
assert len(response.data['results']) == 0
+
+
+def test_library_track_action_import(
+ factories, superuser_api_client, mocker):
+ lt1 = factories['federation.LibraryTrack']()
+ lt2 = factories['federation.LibraryTrack'](library=lt1.library)
+ lt3 = factories['federation.LibraryTrack']()
+ lt4 = factories['federation.LibraryTrack'](library=lt3.library)
+ mocked_run = mocker.patch(
+ 'funkwhale_api.music.tasks.import_batch_run.delay')
+
+ payload = {
+ 'objects': 'all',
+ 'action': 'import',
+ 'filters': {
+ 'library': lt1.library.uuid
+ }
+ }
+ url = reverse('api:v1:federation:library-tracks-action')
+ response = superuser_api_client.post(url, payload, format='json')
+ batch = superuser_api_client.user.imports.latest('id')
+ expected = {
+ 'updated': 2,
+ 'action': 'import',
+ 'result': {
+ 'batch': {'id': batch.pk}
+ }
+ }
+
+ imported_lts = [lt1, lt2]
+ assert response.status_code == 200
+ assert response.data == expected
+ assert batch.jobs.count() == 2
+ for i, job in enumerate(batch.jobs.all()):
+ assert job.library_track == imported_lts[i]
+ mocked_run.assert_called_once_with(import_batch_id=batch.pk)
diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py
index 26cb9453e..dfe649be0 100644
--- a/api/tests/music/test_tasks.py
+++ b/api/tests/music/test_tasks.py
@@ -47,6 +47,15 @@ def test_set_acoustid_on_track_file_required_high_score(factories, mocker):
assert track_file.acoustid_track_id is None
+def test_import_batch_run(factories, mocker):
+ job = factories['music.ImportJob']()
+ mocked_job_run = mocker.patch(
+ 'funkwhale_api.music.tasks.import_job_run.delay')
+ tasks.import_batch_run(import_batch_id=job.batch.pk)
+
+ mocked_job_run.assert_called_once_with(import_job_id=job.pk)
+
+
def test_import_job_can_run_with_file_and_acoustid(
artists, albums, tracks, preferences, factories, mocker):
preferences['providers_acoustid__api_key'] = 'test'
diff --git a/api/tests/music/test_views.py b/api/tests/music/test_views.py
index 38366442f..9328ba329 100644
--- a/api/tests/music/test_views.py
+++ b/api/tests/music/test_views.py
@@ -249,24 +249,6 @@ def test_serve_updates_access_date(factories, settings, api_client):
assert track_file.accessed_date > now
-def test_can_create_import_from_federation_tracks(
- factories, superuser_api_client, mocker):
- lts = factories['federation.LibraryTrack'].create_batch(size=5)
- mocker.patch('funkwhale_api.music.tasks.import_job_run')
-
- payload = {
- 'library_tracks': [l.pk for l in lts]
- }
- url = reverse('api:v1:submit-federation')
- response = superuser_api_client.post(url, payload)
-
- assert response.status_code == 201
- batch = superuser_api_client.user.imports.latest('id')
- assert batch.jobs.count() == 5
- for i, job in enumerate(batch.jobs.all()):
- assert job.library_track == lts[i]
-
-
def test_can_list_import_jobs(factories, superuser_api_client):
job = factories['music.ImportJob']()
url = reverse('api:v1:import-jobs-list')
diff --git a/changes/changelog.d/164.enhancement b/changes/changelog.d/164.enhancement
new file mode 100644
index 000000000..ceea6c2b8
--- /dev/null
+++ b/changes/changelog.d/164.enhancement
@@ -0,0 +1,2 @@
+Can now import a whole remote library at once thanks to new Action Table
+component (#164)
diff --git a/changes/changelog.d/228.feature b/changes/changelog.d/228.feature
new file mode 100644
index 000000000..548c1927e
--- /dev/null
+++ b/changes/changelog.d/228.feature
@@ -0,0 +1,3 @@
+New action table component for quick and efficient batch actions (#228)
+This is implemented on the federated tracks pages, but will be included
+in other pages as well depending on the feedback.
diff --git a/front/package.json b/front/package.json
index 8844e8bee..3dec9c257 100644
--- a/front/package.json
+++ b/front/package.json
@@ -33,7 +33,7 @@
"raven-js": "^3.22.3",
"semantic-ui-css": "^2.2.10",
"showdown": "^1.8.6",
- "vue": "^2.3.3",
+ "vue": "^2.5.16",
"vue-lazyload": "^1.1.4",
"vue-masonry": "^0.10.16",
"vue-router": "^2.3.1",
diff --git a/front/src/components/common/ActionTable.vue b/front/src/components/common/ActionTable.vue
new file mode 100644
index 000000000..718e57b19
--- /dev/null
+++ b/front/src/components/common/ActionTable.vue
@@ -0,0 +1,215 @@
+
+ {{ $t('Do you want to launch action "{% action %}" on {% total %} elements?', {action: currentActionName, total: objectsData.count}) }}
+
+ {{ $t('This may affect a lot of elements, please double check this is really what you want.')}}
+ {{ $t('Launch') }}
+
+
+
+
+
diff --git a/front/src/components/common/DangerousButton.vue b/front/src/components/common/DangerousButton.vue
index 690291d5b..52fcdca61 100644
--- a/front/src/components/common/DangerousButton.vue
+++ b/front/src/components/common/DangerousButton.vue
@@ -13,7 +13,7 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
- |
- ||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|
-
-
-
-
-
+
+
+
+
+ {{ $t('Status') }} |
+ {{ $t('Title') }} |
+ {{ $t('Artist') }} |
+ {{ $t('Album') }} |
+ {{ $t('Published date') }} |
+ {{ $t('Library') }} |
+
+
+
+ {{ $t('In library') }}
+ {{ $t('Import pending') }}
+ {{ $t('Not imported') }}
|
- {{ track.title|truncate(30) }}
+ {{ scope.obj.title|truncate(30) }}
|
- {{ track.artist_name|truncate(30) }}
+ {{ scope.obj.artist_name|truncate(30) }}
|
- {{ track.album_title|truncate(20) }}
+ {{ scope.obj.album_title|truncate(20) }}
|
- |
- {{ track.library.actor.domain }}
+ {{ scope.obj.library.actor.domain }}
|
- |
-
+ |
- - {{ $t('Showing results {%start%}-{%end%} on {%total%}', {start: ((page-1) * paginateBy) + 1 , end: ((page-1) * paginateBy) + result.results.length, total: result.count})}} - |
-
- |
- - | - | - | - |