From f1cf250e2959e2fe7a333994fd10fe1817795e3b Mon Sep 17 00:00:00 2001
From: Eliot Berriot <contact@eliotberriot.com>
Date: Sun, 25 Mar 2018 15:40:37 +0200
Subject: [PATCH] Fix #138: Better handling of utf-8 filenames during file
 import

---
 api/config/settings/common.py       |  1 +
 api/funkwhale_api/common/storage.py | 12 ++++++++++++
 api/tests/files/utf8-éà◌.ogg        |  0
 api/tests/test_import_audio_file.py | 24 ++++++++++++++++++++++++
 changes/changelog.d/138.bugfix      |  1 +
 5 files changed, 38 insertions(+)
 create mode 100644 api/funkwhale_api/common/storage.py
 create mode 100644 api/tests/files/utf8-éà◌.ogg
 create mode 100644 changes/changelog.d/138.bugfix

diff --git a/api/config/settings/common.py b/api/config/settings/common.py
index 077566d1c..f8bd77252 100644
--- a/api/config/settings/common.py
+++ b/api/config/settings/common.py
@@ -231,6 +231,7 @@ STATIC_ROOT = env("STATIC_ROOT", default=str(ROOT_DIR('staticfiles')))
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
 STATIC_URL = env("STATIC_URL", default='/staticfiles/')
+DEFAULT_FILE_STORAGE = 'funkwhale_api.common.storage.ASCIIFileSystemStorage'
 
 # See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
 STATICFILES_DIRS = (
diff --git a/api/funkwhale_api/common/storage.py b/api/funkwhale_api/common/storage.py
new file mode 100644
index 000000000..658ce795a
--- /dev/null
+++ b/api/funkwhale_api/common/storage.py
@@ -0,0 +1,12 @@
+import unicodedata
+
+from django.core.files.storage import FileSystemStorage
+
+
+class ASCIIFileSystemStorage(FileSystemStorage):
+    """
+    Convert unicode characters in name to ASCII characters.
+    """
+    def get_valid_name(self, name):
+        name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore')
+        return super().get_valid_name(name)
diff --git a/api/tests/files/utf8-éà◌.ogg b/api/tests/files/utf8-éà◌.ogg
new file mode 100644
index 000000000..e69de29bb
diff --git a/api/tests/test_import_audio_file.py b/api/tests/test_import_audio_file.py
index 4f3de27db..67263e66d 100644
--- a/api/tests/test_import_audio_file.py
+++ b/api/tests/test_import_audio_file.py
@@ -98,3 +98,27 @@ def test_import_files_skip_acoustid(factories, mocker):
         music_tasks.import_job_run.delay,
         import_job_id=job.pk,
         use_acoustid=False)
+
+
+def test_import_files_works_with_utf8_file_name(factories, mocker):
+    m = mocker.patch('funkwhale_api.common.utils.on_commit')
+    user = factories['users.User'](username='me')
+    path = os.path.join(DATA_DIR, 'utf8-éà◌.ogg')
+    call_command(
+        'import_files',
+        path,
+        username='me',
+        async=True,
+        no_acoustid=True,
+        interactive=False)
+    batch = user.imports.latest('id')
+    job = batch.jobs.first()
+    m.assert_called_once_with(
+        music_tasks.import_job_run.delay,
+        import_job_id=job.pk,
+        use_acoustid=False)
+
+
+def test_storage_rename_utf_8_files(factories):
+    tf = factories['music.TrackFile'](audio_file__filename='été.ogg')
+    assert tf.audio_file.name.endswith('ete.ogg')
diff --git a/changes/changelog.d/138.bugfix b/changes/changelog.d/138.bugfix
new file mode 100644
index 000000000..2a8f7aeb0
--- /dev/null
+++ b/changes/changelog.d/138.bugfix
@@ -0,0 +1 @@
+Better handling of utf-8 filenames during file import (#138)