Merge branch '138-surrogate-error-handling' into 'develop'

Resolve "Importer chokes on files with bad unicode characters"

Closes #138

See merge request funkwhale/funkwhale!309

api/funkwhale_api/providers/audiofile/management/commands/import_files.py

@@ -82,10 +82,31 @@ class Command(BaseCommand):
         try:
             for import_path in options["path"]:
                 matching += glob.glob(import_path, **glob_kwargs)
-            matching = sorted(list(set(matching)))
+            raw_matching = sorted(list(set(matching)))
         except TypeError:
             raise Exception("You need Python 3.5 to use the --recursive flag")
 
+        matching = []
+        for m in raw_matching:
+            # In some situations, the path is encoded incorrectly on the filesystem
+            # so we filter out faulty paths and display a warning to the user.
+            # see https://code.eliotberriot.com/funkwhale/funkwhale/issues/138
+            try:
+                m.encode("utf-8")
+                matching.append(m)
+            except UnicodeEncodeError:
+                try:
+                    previous = matching[-1]
+                except IndexError:
+                    previous = None
+                self.stderr.write(
+                    self.style.WARNING(
+                        "[warning] Ignoring undecodable path. Previous ok file was {}".format(
+                            previous
+                        )
+                    )
+                )
+
         if options["in_place"]:
             self.stdout.write(
                 "Checking imported paths against settings.MUSIC_DIRECTORY_PATH"

changes/changelog.d/138.bugfix

@@ -0,0 +1 @@
+Raise a warning instead of crashing when getting a broken path in file import (#138)