Fix #138: Better handling of utf-8 filenames during file import

api/config/settings/common.py

@@ -231,6 +231,7 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
 STATIC_URL = env("STATIC_URL", default='/staticfiles/')
+DEFAULT_FILE_STORAGE = 'funkwhale_api.common.storage.ASCIIFileSystemStorage'
 
 # See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
 STATICFILES_DIRS = (

api/funkwhale_api/common/storage.py

@@ -0,0 +1,12 @@
+import unicodedata
+
+from django.core.files.storage import FileSystemStorage
+
+
+class ASCIIFileSystemStorage(FileSystemStorage):
+    """
+    Convert unicode characters in name to ASCII characters.
+    """
+    def get_valid_name(self, name):
+        name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore')
+        return super().get_valid_name(name)

api/tests/test_import_audio_file.py

@@ -98,3 +98,27 @@ def test_import_files_skip_acoustid(factories, mocker):
         music_tasks.import_job_run.delay,
         import_job_id=job.pk,
         use_acoustid=False)
+
+
+def test_import_files_works_with_utf8_file_name(factories, mocker):
+    m = mocker.patch('funkwhale_api.common.utils.on_commit')
+    user = factories['users.User'](username='me')
+    path = os.path.join(DATA_DIR, 'utf8-éà◌.ogg')
+    call_command(
+        'import_files',
+        path,
+        username='me',
+        async=True,
+        no_acoustid=True,
+        interactive=False)
+    batch = user.imports.latest('id')
+    job = batch.jobs.first()
+    m.assert_called_once_with(
+        music_tasks.import_job_run.delay,
+        import_job_id=job.pk,
+        use_acoustid=False)
+
+
+def test_storage_rename_utf_8_files(factories):
+    tf = factories['music.TrackFile'](audio_file__filename='été.ogg')
+    assert tf.audio_file.name.endswith('ete.ogg')

changes/changelog.d/138.bugfix

@@ -0,0 +1 @@
+Better handling of utf-8 filenames during file import (#138)