kopia lustrzana https://dev.funkwhale.audio/funkwhale/funkwhale
Brand new file importer
Eliot Berriot
rodzic
2e616282fd
commit
1c8f055490
10
CHANGELOG
10
CHANGELOG
|
|
@ -2,7 +2,15 @@ Changelog
|
|||
=========
|
||||
|
||||
|
||||
0.2.7 (Unreleased)
|
||||
0.3 (Unreleased)
|
||||
------------------
|
||||
|
||||
- Revamped all import logic, everything is more tested and consistend
|
||||
- Can now use Acoustid in file imports to automatically grab metadata from musicbrainz
|
||||
- Brand new file import wizard
|
||||
|
||||
|
||||
0.2.7
|
||||
------------------
|
||||
|
||||
- Shortcuts: can now use the ``f`` shortcut to toggle the currently playing track
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ router.register(r'trackfiles', views.TrackFileViewSet, 'trackfiles')
|
|||
router.register(r'artists', views.ArtistViewSet, 'artists')
|
||||
router.register(r'albums', views.AlbumViewSet, 'albums')
|
||||
router.register(r'import-batches', views.ImportBatchViewSet, 'import-batches')
|
||||
router.register(r'import-jobs', views.ImportJobViewSet, 'import-jobs')
|
||||
router.register(r'submit', views.SubmitViewSet, 'submit')
|
||||
router.register(r'playlists', playlists_views.PlaylistViewSet, 'playlists')
|
||||
router.register(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from django.conf import settings
|
||||
|
||||
from rest_framework.permissions import BasePermission
|
||||
from rest_framework.permissions import BasePermission, DjangoModelPermissions
|
||||
|
||||
|
||||
class ConditionalAuthentication(BasePermission):
|
||||
|
|
@ -9,3 +9,14 @@ class ConditionalAuthentication(BasePermission):
|
|||
if settings.API_AUTHENTICATION_REQUIRED:
|
||||
return request.user and request.user.is_authenticated
|
||||
return True
|
||||
|
||||
|
||||
class HasModelPermission(DjangoModelPermissions):
|
||||
"""
|
||||
Same as DjangoModelPermissions, but we pin the model:
|
||||
|
||||
class MyModelPermission(HasModelPermission):
|
||||
model = User
|
||||
"""
|
||||
def get_required_permissions(self, method, model_cls):
|
||||
return super().get_required_permissions(method, self.model)
|
||||
|
|
|
|||
|
|
@ -405,8 +405,11 @@ class ImportBatch(models.Model):
|
|||
@property
|
||||
def status(self):
|
||||
pending = any([job.status == 'pending' for job in self.jobs.all()])
|
||||
errored = any([job.status == 'errored' for job in self.jobs.all()])
|
||||
if pending:
|
||||
return 'pending'
|
||||
if errored:
|
||||
return 'errored'
|
||||
return 'finished'
|
||||
|
||||
class ImportJob(models.Model):
|
||||
|
|
@ -418,7 +421,7 @@ class ImportJob(models.Model):
|
|||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.CASCADE)
|
||||
source = models.URLField()
|
||||
source = models.CharField(max_length=500)
|
||||
mbid = models.UUIDField(editable=False, null=True, blank=True)
|
||||
STATUS_CHOICES = (
|
||||
('pending', 'Pending'),
|
||||
|
|
|
|||
|
|
@ -113,7 +113,8 @@ class ImportJobSerializer(serializers.ModelSerializer):
|
|||
track_file = TrackFileSerializer(read_only=True)
|
||||
class Meta:
|
||||
model = models.ImportJob
|
||||
fields = ('id', 'mbid', 'source', 'status', 'track_file')
|
||||
fields = ('id', 'mbid', 'batch', 'source', 'status', 'track_file', 'audio_file')
|
||||
read_only_fields = ('status', 'track_file')
|
||||
|
||||
|
||||
class ImportBatchSerializer(serializers.ModelSerializer):
|
||||
|
|
@ -121,3 +122,4 @@ class ImportBatchSerializer(serializers.ModelSerializer):
|
|||
class Meta:
|
||||
model = models.ImportBatch
|
||||
fields = ('id', 'jobs', 'status', 'creation_date')
|
||||
read_only_fields = ('creation_date',)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from django.core.files.base import ContentFile
|
|||
|
||||
from funkwhale_api.taskapp import celery
|
||||
from funkwhale_api.providers.acoustid import get_acoustid_client
|
||||
from funkwhale_api.providers.audiofile.tasks import import_track_data_from_path
|
||||
|
||||
from django.conf import settings
|
||||
from . import models
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from django.urls import reverse
|
|||
from django.db import models, transaction
|
||||
from django.db.models.functions import Length
|
||||
from django.conf import settings
|
||||
from rest_framework import viewsets, views
|
||||
from rest_framework import viewsets, views, mixins
|
||||
from rest_framework.decorators import detail_route, list_route
|
||||
from rest_framework.response import Response
|
||||
from rest_framework import permissions
|
||||
|
|
@ -15,7 +15,8 @@ from django.contrib.auth.decorators import login_required
|
|||
from django.utils.decorators import method_decorator
|
||||
|
||||
from funkwhale_api.musicbrainz import api
|
||||
from funkwhale_api.common.permissions import ConditionalAuthentication
|
||||
from funkwhale_api.common.permissions import (
|
||||
ConditionalAuthentication, HasModelPermission)
|
||||
from taggit.models import Tag
|
||||
|
||||
from . import models
|
||||
|
|
@ -71,16 +72,45 @@ class AlbumViewSet(SearchMixin, viewsets.ReadOnlyModelViewSet):
|
|||
ordering_fields = ('creation_date',)
|
||||
|
||||
|
||||
class ImportBatchViewSet(viewsets.ReadOnlyModelViewSet):
|
||||
class ImportBatchViewSet(
|
||||
mixins.CreateModelMixin,
|
||||
mixins.ListModelMixin,
|
||||
mixins.RetrieveModelMixin,
|
||||
viewsets.GenericViewSet):
|
||||
queryset = (
|
||||
models.ImportBatch.objects.all()
|
||||
.prefetch_related('jobs__track_file')
|
||||
.order_by('-creation_date'))
|
||||
serializer_class = serializers.ImportBatchSerializer
|
||||
permission_classes = (permissions.DjangoModelPermissions, )
|
||||
|
||||
def get_queryset(self):
|
||||
return super().get_queryset().filter(submitted_by=self.request.user)
|
||||
|
||||
def perform_create(self, serializer):
|
||||
serializer.save(submitted_by=self.request.user)
|
||||
|
||||
|
||||
class ImportJobPermission(HasModelPermission):
|
||||
# not a typo, perms on import job is proxied to import batch
|
||||
model = models.ImportBatch
|
||||
|
||||
|
||||
class ImportJobViewSet(
|
||||
mixins.CreateModelMixin,
|
||||
viewsets.GenericViewSet):
|
||||
queryset = (models.ImportJob.objects.all())
|
||||
serializer_class = serializers.ImportJobSerializer
|
||||
permission_classes = (ImportJobPermission, )
|
||||
|
||||
def get_queryset(self):
|
||||
return super().get_queryset().filter(batch__submitted_by=self.request.user)
|
||||
|
||||
def perform_create(self, serializer):
|
||||
source = 'file://' + serializer.validated_data['audio_file'].name
|
||||
serializer.save(source=source)
|
||||
tasks.import_job_run.delay(import_job_id=serializer.instance.pk)
|
||||
|
||||
|
||||
class TrackViewSet(TagViewSetMixin, SearchMixin, viewsets.ReadOnlyModelViewSet):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
import pytest
|
||||
from django.urls import reverse
|
||||
|
||||
|
|
@ -8,6 +9,8 @@ from funkwhale_api.music import serializers
|
|||
|
||||
from . import data as api_data
|
||||
|
||||
DATA_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def test_can_submit_youtube_url_for_track_import(mocker, superuser_client):
|
||||
mocker.patch(
|
||||
|
|
@ -189,6 +192,48 @@ def test_user_can_query_api_for_his_own_batches(client, factories):
|
|||
assert results['results'][0]['jobs'][0]['mbid'] == job.mbid
|
||||
|
||||
|
||||
def test_user_can_create_an_empty_batch(client, factories):
|
||||
user = factories['users.SuperUser']()
|
||||
url = reverse('api:v1:import-batches-list')
|
||||
client.login(username=user.username, password='test')
|
||||
response = client.post(url)
|
||||
|
||||
assert response.status_code == 201
|
||||
|
||||
batch = user.imports.latest('id')
|
||||
|
||||
assert batch.submitted_by == user
|
||||
assert batch.source == 'api'
|
||||
|
||||
|
||||
def test_user_can_create_import_job_with_file(client, factories, mocker):
|
||||
path = os.path.join(DATA_DIR, 'test.ogg')
|
||||
m = mocker.patch('funkwhale_api.music.tasks.import_job_run.delay')
|
||||
user = factories['users.SuperUser']()
|
||||
batch = factories['music.ImportBatch'](submitted_by=user)
|
||||
url = reverse('api:v1:import-jobs-list')
|
||||
client.login(username=user.username, password='test')
|
||||
with open(path, 'rb') as f:
|
||||
content = f.read()
|
||||
f.seek(0)
|
||||
response = client.post(url, {
|
||||
'batch': batch.pk,
|
||||
'audio_file': f,
|
||||
'source': 'file://'
|
||||
}, format='multipart')
|
||||
|
||||
assert response.status_code == 201
|
||||
|
||||
job = batch.jobs.latest('id')
|
||||
|
||||
assert job.status == 'pending'
|
||||
assert job.source.startswith('file://')
|
||||
assert 'test.ogg' in job.source
|
||||
assert job.audio_file.read() == content
|
||||
|
||||
m.assert_called_once_with(import_job_id=job.pk)
|
||||
|
||||
|
||||
def test_can_search_artist(factories, client):
|
||||
artist1 = factories['music.Artist']()
|
||||
artist2 = factories['music.Artist']()
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ server {
|
|||
rewrite ^(.+)$ /index.html last;
|
||||
}
|
||||
location /api/ {
|
||||
# this is needed if you have file import via upload enabled
|
||||
client_max_body_size 30M;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ http {
|
|||
server {
|
||||
listen 6001;
|
||||
charset utf-8;
|
||||
|
||||
client_max_body_size 20M;
|
||||
location /_protected/media {
|
||||
internal;
|
||||
alias /protected/media;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
"vue-lazyload": "^1.1.4",
|
||||
"vue-resource": "^1.3.4",
|
||||
"vue-router": "^2.3.1",
|
||||
"vue-upload-component": "^2.7.4",
|
||||
"vuedraggable": "^2.14.1",
|
||||
"vuex": "^3.0.1",
|
||||
"vuex-persistedstate": "^2.4.2"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
['ui',
|
||||
{'active': batch.status === 'pending'},
|
||||
{'warning': batch.status === 'pending'},
|
||||
{'error': batch.status === 'errored'},
|
||||
{'success': batch.status === 'finished'},
|
||||
'progress']">
|
||||
<div class="bar" :style="progressBarStyle">
|
||||
|
|
@ -37,7 +38,7 @@
|
|||
</td>
|
||||
<td>
|
||||
<span
|
||||
:class="['ui', {'yellow': job.status === 'pending'}, {'green': job.status === 'finished'}, 'label']">{{ job.status }}</span>
|
||||
:class="['ui', {'yellow': job.status === 'pending'}, {'red': job.status === 'errored'}, {'green': job.status === 'finished'}, 'label']">{{ job.status }}</span>
|
||||
</td>
|
||||
<td>
|
||||
<router-link v-if="job.track_file" :to="{name: 'library.tracks.detail', params: {id: job.track_file.track }}">{{ job.track_file.track }}</router-link>
|
||||
|
|
@ -89,7 +90,7 @@ export default {
|
|||
computed: {
|
||||
progress () {
|
||||
return this.batch.jobs.filter(j => {
|
||||
return j.status === 'finished'
|
||||
return j.status !== 'pending'
|
||||
}).length * 100 / this.batch.jobs.length
|
||||
},
|
||||
progressBarStyle () {
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
<td>{{ result.jobs.length }}</td>
|
||||
<td>
|
||||
<span
|
||||
:class="['ui', {'yellow': result.status === 'pending'}, {'green': result.status === 'finished'}, 'label']">{{ result.status }}</span>
|
||||
:class="['ui', {'yellow': result.status === 'pending'}, {'red': result.status === 'errored'}, {'green': result.status === 'finished'}, 'label']">{{ result.status }}</span>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
<template>
|
||||
<div>
|
||||
<div v-if="batch" class="ui two buttons">
|
||||
<file-upload-widget
|
||||
class="ui icon button"
|
||||
:post-action="uploadUrl"
|
||||
:multiple="true"
|
||||
:size="1024 * 1024 * 30"
|
||||
:data="uploadData"
|
||||
:drop="true"
|
||||
extensions="ogg,mp3"
|
||||
accept="audio/*"
|
||||
v-model="files"
|
||||
name="audio_file"
|
||||
:thread="3"
|
||||
@input-filter="inputFilter"
|
||||
@input-file="inputFile"
|
||||
ref="upload">
|
||||
<i class="upload icon"></i>
|
||||
Select files to upload...
|
||||
</file-upload-widget>
|
||||
<button class="ui icon teal button" v-if="!$refs.upload || !$refs.upload.active" @click.prevent="$refs.upload.active = true">
|
||||
<i class="play icon" aria-hidden="true"></i>
|
||||
Start Upload
|
||||
</button>
|
||||
<button type="button" class="ui icon yellow button" v-else @click.prevent="$refs.upload.active = false">
|
||||
<i class="pause icon" aria-hidden="true"></i>
|
||||
Stop Upload
|
||||
</button>
|
||||
</div>
|
||||
<div class="ui hidden divider"></div>
|
||||
<p>
|
||||
Once all your files are uploaded, simply head over <router-link :to="{name: 'library.import.batches.detail', params: {id: batch.id }}">import detail page</router-link> to check the import status.
|
||||
</p>
|
||||
<table class="ui single line table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>File name</th>
|
||||
<th>Size</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="(file, index) in files" :key="file.id">
|
||||
<td>{{ file.name }}</td>
|
||||
<td>{{ file.size }}</td>
|
||||
<td>
|
||||
<span v-if="file.error" class="ui red label">
|
||||
{{ file.error }}
|
||||
</span>
|
||||
<span v-else-if="file.success" class="ui green label">Success</span>
|
||||
<span v-else-if="file.active" class="ui yellow label">Uploading...</span>
|
||||
<template v-else>
|
||||
<span class="ui label">Pending</span>
|
||||
<button class="ui tiny basic red icon button" @click.prevent="$refs.upload.remove(file)"><i class="delete icon"></i></button>
|
||||
</template>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import Vue from 'vue'
|
||||
import logger from '@/logging'
|
||||
import FileUploadWidget from './FileUploadWidget'
|
||||
import config from '@/config'
|
||||
|
||||
export default {
|
||||
components: {
|
||||
FileUploadWidget
|
||||
},
|
||||
data () {
|
||||
return {
|
||||
files: [],
|
||||
uploadUrl: config.API_URL + 'import-jobs/',
|
||||
batch: null
|
||||
}
|
||||
},
|
||||
mounted: function () {
|
||||
this.createBatch()
|
||||
},
|
||||
methods: {
|
||||
inputFilter (newFile, oldFile, prevent) {
|
||||
if (newFile && !oldFile) {
|
||||
let extension = newFile.name.split('.').pop()
|
||||
if (['ogg', 'mp3'].indexOf(extension) < 0) {
|
||||
prevent()
|
||||
}
|
||||
}
|
||||
},
|
||||
inputFile (newFile, oldFile) {
|
||||
if (newFile && !oldFile) {
|
||||
// add
|
||||
console.log('add', newFile)
|
||||
if (!this.batch) {
|
||||
this.createBatch()
|
||||
}
|
||||
}
|
||||
if (newFile && oldFile) {
|
||||
// update
|
||||
console.log('update', newFile)
|
||||
}
|
||||
if (!newFile && oldFile) {
|
||||
// remove
|
||||
console.log('remove', oldFile)
|
||||
}
|
||||
},
|
||||
createBatch () {
|
||||
let self = this
|
||||
let url = config.API_URL + 'import-batches/'
|
||||
let resource = Vue.resource(url)
|
||||
resource.save({}, {}).then((response) => {
|
||||
self.batch = response.data
|
||||
}, (response) => {
|
||||
logger.default.error('error while launching creating batch')
|
||||
})
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
batchId: function () {
|
||||
if (this.batch) {
|
||||
return this.batch.id
|
||||
}
|
||||
return null
|
||||
},
|
||||
uploadData: function () {
|
||||
return {
|
||||
'batch': this.batchId,
|
||||
'source': 'file://'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<!-- Add "scoped" attribute to limit CSS to this component only -->
|
||||
<style scoped>
|
||||
</style>
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
<script>
|
||||
import FileUpload from 'vue-upload-component'
|
||||
|
||||
export default {
|
||||
extends: FileUpload,
|
||||
methods: {
|
||||
uploadHtml5 (file) {
|
||||
let form = new window.FormData()
|
||||
let value
|
||||
for (let key in file.data) {
|
||||
value = file.data[key]
|
||||
if (value && typeof value === 'object' && typeof value.toString !== 'function') {
|
||||
if (value instanceof File) {
|
||||
form.append(key, value, value.name)
|
||||
} else {
|
||||
form.append(key, JSON.stringify(value))
|
||||
}
|
||||
} else if (value !== null && value !== undefined) {
|
||||
form.append(key, value)
|
||||
}
|
||||
}
|
||||
form.append(this.name, file.file, file.file.filename || file.name)
|
||||
let xhr = new XMLHttpRequest()
|
||||
xhr.open('POST', file.postAction)
|
||||
xhr.setRequestHeader('Authorization', this.$store.getters['auth/header'])
|
||||
return this.uploadXhr(xhr, file, form)
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<!-- Add "scoped" attribute to limit CSS to this component only -->
|
||||
<style scoped>
|
||||
</style>
|
||||
|
|
@ -39,8 +39,8 @@
|
|||
</div>
|
||||
</div>
|
||||
<div class="field">
|
||||
<div class="ui disabled radio checkbox">
|
||||
<input type="radio" id="upload" value="upload" v-model="currentSource" disabled>
|
||||
<div class="ui radio checkbox">
|
||||
<input type="radio" id="upload" value="upload" v-model="currentSource">
|
||||
<label for="upload">File upload</label>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -84,8 +84,14 @@
|
|||
</div>
|
||||
</div>
|
||||
<div v-if="currentStep === 2">
|
||||
<file-upload
|
||||
ref="import"
|
||||
v-if="currentSource == 'upload'"
|
||||
></file-upload>
|
||||
|
||||
<component
|
||||
ref="import"
|
||||
v-if="currentSource == 'external'"
|
||||
:metadata="metadata"
|
||||
:is="importComponent"
|
||||
:backends="backends"
|
||||
|
|
@ -119,6 +125,7 @@ import MetadataSearch from '@/components/metadata/Search'
|
|||
import ReleaseCard from '@/components/metadata/ReleaseCard'
|
||||
import ArtistCard from '@/components/metadata/ArtistCard'
|
||||
import ReleaseImport from './ReleaseImport'
|
||||
import FileUpload from './FileUpload'
|
||||
import ArtistImport from './ArtistImport'
|
||||
|
||||
import router from '@/router'
|
||||
|
|
@ -130,7 +137,8 @@ export default {
|
|||
ArtistCard,
|
||||
ReleaseCard,
|
||||
ArtistImport,
|
||||
ReleaseImport
|
||||
ReleaseImport,
|
||||
FileUpload
|
||||
},
|
||||
props: {
|
||||
mbType: {type: String, required: false},
|
||||
|
|
@ -142,7 +150,7 @@ export default {
|
|||
currentType: this.mbType || 'artist',
|
||||
currentId: this.mbId,
|
||||
currentStep: 0,
|
||||
currentSource: this.source || 'external',
|
||||
currentSource: '',
|
||||
metadata: {},
|
||||
isImporting: false,
|
||||
importData: {
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue