add favorite sync

Part-of: <https://dev.funkwhale.audio/funkwhale/funkwhale/-/merge_requests/2658>

api/config/settings/common.py

@@ -950,6 +950,16 @@ CELERY_BEAT_SCHEDULE = {
         ),
         "options": {"expires": 60 * 60},
     },
+    "listenbrainz.trigger_listening_sync_with_listenbrainz": {
+        "task": "listenbrainz.trigger_listening_sync_with_listenbrainz",
+        "schedule": crontab(day_of_week="*", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    },
+    "listenbrainz.trigger_favorite_sync_with_listenbrainz": {
+        "task": "listenbrainz.trigger_favorite_sync_with_listenbrainz",
+        "schedule": crontab(day_of_week="*", minute="0", hour="3"),
+        "options": {"expires": 60 * 60 * 24},
+    },
 }
 
 if env.str("TYPESENSE_API_KEY", default=None):

api/funkwhale_api/contrib/listenbrainz/funkwhale_ready.py

@@ -1,14 +1,15 @@
-import liblistenbrainz
-from django.utils import timezone
-
 import funkwhale_api
-import pylistenbrainz
+import liblistenbrainz
 
-from .funkwhale_startup import PLUGIN
+from config import plugins
+from django.utils import timezone
 
 from funkwhale_api.history import models as history_models
 from funkwhale_api.favorites import models as favorites_models
 
+from .funkwhale_startup import PLUGIN
+from . import tasks
+
 
 @plugins.register_hook(plugins.LISTENING_CREATED, PLUGIN)
 def submit_listen(listening, conf, **kwargs):
@@ -20,12 +21,12 @@ def submit_listen(listening, conf, **kwargs):
     logger.info("Submitting listen to ListenBrainz")
     client = liblistenbrainz.ListenBrainz()
     client.set_auth_token(user_token)
-    listen = get_listen(listening.track)
+    listen = get_lb_listen(listening)
 
     client.submit_single_listen(listen)
 
 
-def get_listen(listening):
+def get_lb_listen(listening):
     track = listening.track
     additional_info = {
         "media_player": "Funkwhale",
@@ -68,11 +69,11 @@ def submit_favorite_creation(track_favorite, conf, **kwargs):
         return
     logger = PLUGIN["logger"]
     logger.info("Submitting favorite to ListenBrainz")
-    client = pylistenbrainz.ListenBrainz()
+    client = liblistenbrainz.ListenBrainz()
-    track = get_listen(track_favorite.track)
+    track = track_favorite.track
     if not track.mbid:
         logger.warning(
-            "This tracks doesn't have a mbid. Feedback will not be sublited to Listenbrainz"
+            "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz"
         )
         return
     client.submit_user_feedback(1, track.mbid)
@@ -85,8 +86,8 @@ def submit_favorite_deletion(track_favorite, conf, **kwargs):
         return
     logger = PLUGIN["logger"]
     logger.info("Submitting favorite deletion to ListenBrainz")
-    client = pylistenbrainz.ListenBrainz()
+    client = liblistenbrainz.ListenBrainz()
-    track = get_listen(track_favorite.track)
+    track = track_favorite.track
     if not track.mbid:
         logger.warning(
             "This tracks doesn't have a mbid. Feedback will not be submited to Listenbrainz"
@@ -109,13 +110,12 @@ def sync_listenings_from_listenbrainz(user, conf):
             .filter(source="Listenbrainz")
             .latest("creation_date")
             .values_list("creation_date", flat=True)
-        )
+        ).timestamp()
-        last_ts.timestamp()
+    except funkwhale_api.history.models.Listening.DoesNotExist:
-    except history_models.Listening.DoesNotExist:
+        tasks.import_listenbrainz_listenings(user, user_name, 0)
-        tasks.import_listenbrainz_listenings(user, user_name, ts=0)
         return
 
-    tasks.import_listenbrainz_listenings(user, user_name, ts=last_ts)
+    tasks.import_listenbrainz_listenings(user, user_name, last_ts)
 
 
 @plugins.register_hook(plugins.FAVORITE_SYNC, PLUGIN)
@@ -129,11 +129,10 @@ def sync_favorites_from_listenbrainz(user, conf):
             favorites_models.TrackFavorite.objects.filter(user=user)
             .filter(source="Listenbrainz")
             .latest("creation_date")
-            .values_list("creation_date", flat=True)
+            .creation_date.timestamp()
         )
-        last_ts.timestamp()
+    except favorites_models.TrackFavorite.DoesNotExist:
-    except history_models.Listening.DoesNotExist:
+        tasks.import_listenbrainz_favorites(user, user_name, 0)
-        tasks.import_listenbrainz_favorites(user, user_name, ts=0)
         return
 
-    tasks.import_listenbrainz_favorites(user, user_name, ts=last_ts)
+    tasks.import_listenbrainz_favorites(user, user_name, last_ts)

api/funkwhale_api/contrib/listenbrainz/tasks.py

@@ -1,5 +1,5 @@
 import datetime
-import pylistenbrainz
+import liblistenbrainz
 
 from django.utils import timezone
 from config import plugins
@@ -47,27 +47,20 @@ def trigger_favorite_sync_with_listenbrainz():
 
 
 @celery.app.task(name="listenbrainz.import_listenbrainz_listenings")
-def import_listenbrainz_listenings(user, user_name, ts):
+def import_listenbrainz_listenings(user, user_name, since):
-    client = pylistenbrainz.ListenBrainz()
+    client = liblistenbrainz.ListenBrainz()
-    listens = client.get_listens(username=user_name, min_ts=ts, count=100)
+    response = client.get_listens(username=user_name, min_ts=since, count=100)
-    add_lb_listenings_to_db(listens, user)
+    listens = response["payload"]["listens"]
-    new_ts = 13
+    while listens:
-    last_ts = 12
+        add_lb_listenings_to_db(listens, user)
-    while new_ts != last_ts:
-        last_ts = max(
-            listens,
-            key=lambda obj: datetime.datetime.fromtimestamp(
-                obj.listened_at, timezone.utc
-            ),
-        )
-        listens = client.get_listens(username=user_name, min_ts=new_ts, count=100)
         new_ts = max(
             listens,
             key=lambda obj: datetime.datetime.fromtimestamp(
                 obj.listened_at, timezone.utc
             ),
         )
-        add_lb_listenings_to_db(listens, user)
+        response = client.get_listens(username=user_name, min_ts=new_ts, count=100)
+        listens = response["payload"]["listens"]
 
 
 def add_lb_listenings_to_db(listens, user):
@@ -119,46 +112,51 @@ def add_lb_listenings_to_db(listens, user):
 
 
 @celery.app.task(name="listenbrainz.import_listenbrainz_favorites")
-def import_listenbrainz_favorites(user, user_name, last_sync):
+def import_listenbrainz_favorites(user, user_name, since):
-    client = pylistenbrainz.ListenBrainz()
+    client = liblistenbrainz.ListenBrainz()
-    last_ts = int(datetime.datetime.now(timezone.utc).timestamp())
+    response = client.get_user_feedback(username=user_name)
     offset = 0
-    while last_ts >= last_sync:
+    while response["feedback"]:
-        feedbacks = client.get_user_feedback(username=user_name, offset=offset)
+        count = response["count"]
-        add_lb_feedback_to_db(feedbacks, user)
+        offset = offset + count
-        offset = feedbacks.count
+        last_sync = min(
-        last_ts = max(
+            response["feedback"],
-            feedbacks.feedback,
+            key=lambda obj: datetime.datetime.fromtimestamp(
-            key=lambda obj: datetime.datetime.fromtimestamp(obj.created, timezone.utc),
+                obj["created"], timezone.utc
-        )
+            ),
-        # to do implement offset in pylb
+        )["created"]
+        add_lb_feedback_to_db(response["feedback"], user)
+        if last_sync <= since or count == 0:
+            return
+        response = client.get_user_feedback(username=user_name, offset=offset)
 
 
 def add_lb_feedback_to_db(feedbacks, user):
     logger = PLUGIN["logger"]
-    fw_listens = []
+    for feedback in feedbacks:
-    for feedback in feedbacks.feedback:
         try:
-            track = music_models.Track.objects.get(mbid=feedback.recording_mbid)
+            track = music_models.Track.objects.get(mbid=feedback["recording_mbid"])
         except music_models.Track.DoesNotExist:
             logger.info(
                 "Received feedback track doesn't exist in fw database. Skipping..."
             )
             continue
 
-        if feedback.score == 1:
+        if feedback["score"] == 1:
             favorites_models.TrackFavorite.objects.get_or_create(
                 user=user,
                 creation_date=datetime.datetime.fromtimestamp(
-                    feedback.created, timezone.utc
+                    feedback["created"], timezone.utc
                 ),
                 track=track,
                 source="Listenbrainz",
             )
-        elif feedback.score == 0:
+        elif feedback["score"] == 0:
             try:
-                favorites_models.TrackFavorite.objects.delete(user=user, track=track)
+                favorites_models.TrackFavorite.objects.get(
+                    user=user, track=track
+                ).delete()
             except favorites_models.TrackFavorite.DoesNotExist:
                 continue
-        elif feedback.score == -1:
+        elif feedback["score"] == -1:
             logger.info("Funkwhale doesn't support hate yet <3")

api/funkwhale_api/favorites/factories.py

@@ -1,4 +1,5 @@
 import factory
+from django.utils import timezone
 
 from funkwhale_api.factories import NoUpdateOnCreate, registry
 from funkwhale_api.music.factories import TrackFactory
@@ -9,6 +10,7 @@ from funkwhale_api.users.factories import UserFactory
 class TrackFavorite(NoUpdateOnCreate, factory.django.DjangoModelFactory):
     track = factory.SubFactory(TrackFactory)
     user = factory.SubFactory(UserFactory)
+    creation_date = factory.Faker("date_time_this_decade", tzinfo=timezone.utc)
 
     class Meta:
         model = "favorites.TrackFavorite"

api/tests/conftest.py

@@ -277,16 +277,6 @@ def disabled_musicbrainz(mocker):
     )
 
 
-# @pytest.fixture()
-# def disabled_listenbrainz(mocker):
-#     # we ensure no listenbrainz requests gets out
-#     yield mocker.patch.object(
-#         listenbrainz.client.ListenBrainzClient,
-#         "_submit",
-#         return_value=None,
-#     )
-
-
 @pytest.fixture(autouse=True)
 def r_mock(requests_mock):
     """

api/tests/contrib/listenbrainz/test_listenbrainz.py

@@ -2,7 +2,7 @@ import datetime
 import logging
 
 import pytest
-import pylistenbrainz
+import liblistenbrainz
 
 from django.urls import reverse
 from django.utils import timezone
@@ -62,32 +62,47 @@ def test_sync_listenings_from_listenbrainz(factories, mocker, caplog):
         "sync_listenings": True,
     }
 
-    listens = [
+    listens = {
-        pylistenbrainz.Listen(
+        "payload": {
-            track_name="test",
+            "count": 25,
-            artist_name="artist_test",
+            "user_id": "-- the MusicBrainz ID of the user --",
-            recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
+            "listens": [
-            additional_info={"submission_client": "not funkwhale"},
+                liblistenbrainz.Listen(
-            listened_at=-3124224000,
+                    track_name="test",
-        ),
+                    artist_name="artist_test",
-        pylistenbrainz.Listen(
+                    recording_mbid="f89db7f8-4a1f-4228-a0a1-e7ba028b7476",
-            track_name="test2",
+                    additional_info={"submission_client": "not funkwhale"},
-            artist_name="artist_test2",
+                    listened_at=-3124224000,
-            recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
+                ),
-            additional_info={"submission_client": "Funkwhale ListenBrainz plugin"},
+                liblistenbrainz.Listen(
-            listened_at=1871,
+                    track_name="test2",
-        ),
+                    artist_name="artist_test2",
-        pylistenbrainz.Listen(
+                    recording_mbid="54c60860-f43d-484e-b691-7ab7ec8de559",
-            track_name="test3",
+                    additional_info={
-            artist_name="artist_test3",
+                        "submission_client": "Funkwhale ListenBrainz plugin"
-            listened_at=0,
+                    },
-        ),
+                    listened_at=1871,
-    ]
+                ),
+                liblistenbrainz.Listen(
+                    track_name="test3",
+                    artist_name="artist_test3",
+                    listened_at=0,
+                ),
+            ],
+        }
+    }
+    no_more_listen = {
+        "payload": {
+            "count": 25,
+            "user_id": "Bilbo",
+            "listens": [],
+        }
+    }
 
     mocker.patch.object(
-        funkwhale_ready.tasks.pylistenbrainz.ListenBrainz,
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
         "get_listens",
-        return_value=listens,
+        side_effect=[listens, no_more_listen],
     )
 
     funkwhale_ready.sync_listenings_from_listenbrainz(user, conf)
@@ -105,13 +120,20 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
     caplog.set_level(logging.INFO)
     logger.addHandler(caplog.handler)
     user = factories["users.User"]()
-
+    # track lb fav
     factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
-
+    # random track
-    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    factories["music.Track"]()
+    # track lb neutral
     track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
-
+    favorite = factories["favorites.TrackFavorite"](track=track, user=user)
-    favorite = factories["favorites.TrackFavorite"](track=track)
+    # last_sync
+    track_last_sync = factories["music.Track"](
+        mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7"
+    )
+    favorite_last_sync = factories["favorites.TrackFavorite"](
+        track=track_last_sync, source="Listenbrainz"
+    )
 
     conf = {
         "user_name": user.username,
@@ -135,6 +157,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
                 "user_id": user.username,
             },
             {
+                # last sync
                 "created": 1690775094,
                 "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
                 "score": -1,
@@ -142,7 +165,7 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
             },
             {
                 "created": 1690775093,
-                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2 ",
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2",
                 "score": 1,
                 "user_id": user.username,
             },
@@ -150,10 +173,11 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
         "offset": 0,
         "total_count": 4,
     }
+    empty_feedback = {"count": 0, "feedback": [], "offset": 0, "total_count": 0}
     mocker.patch.object(
-        funkwhale_ready.tasks.pylistenbrainz.ListenBrainz,
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
         "get_user_feedback",
-        return_value=feedbacks,
+        side_effect=[feedbacks, empty_feedback],
     )
 
     funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
@@ -161,5 +185,153 @@ def test_sync_favorites_from_listenbrainz(factories, mocker, caplog):
     assert favorites_models.TrackFavorite.objects.filter(
         track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
     ).exists()
-    with pytest.raises(deleted.DoesNotExist):
+    with pytest.raises(favorites_models.TrackFavorite.DoesNotExist):
         favorite.refresh_from_db()
+
+
+def test_sync_favorites_from_listenbrainz_since(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+    # track lb fav
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+    # track lb neutral
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+    favorite = factories["favorites.TrackFavorite"](track=track, user=user)
+    # track should be not synced
+    factories["music.Track"](mbid="1fd02cf2-7247-4715-8862-c378ec196000")
+    # last_sync
+    track_last_sync = factories["music.Track"](
+        mbid="c878ef2f-c08d-4a81-a047-f2a9f978cec7"
+    )
+    factories["favorites.TrackFavorite"](
+        track=track_last_sync,
+        user=user,
+        source="Listenbrainz",
+        creation_date=datetime.datetime.fromtimestamp(1690775094),
+    )
+
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "sync_favorites": True,
+    }
+
+    feedbacks = {
+        "count": 5,
+        "feedback": [
+            {
+                "created": 1701116226,
+                "recording_mbid": "195565db-65f9-4d0d-b347-5f0c85509528",
+                "score": 1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1701116214,
+                "recording_mbid": "c5af5351-dbbf-4481-b52e-a480b6c57986",
+                "score": 0,
+                "user_id": user.username,
+            },
+            {
+                # last sync
+                "created": 1690775094,
+                "recording_mbid": "c878ef2f-c08d-4a81-a047-f2a9f978cec7",
+                "score": -1,
+                "user_id": user.username,
+            },
+            {
+                "created": 1690775093,
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec1965d2",
+                "score": 1,
+                "user_id": user.username,
+            },
+        ],
+        "offset": 0,
+        "total_count": 4,
+    }
+    second_feedback = {
+        "count": 0,
+        "feedback": [
+            {
+                "created": 0,
+                "recording_mbid": "1fd02cf2-7247-4715-8862-c378ec196000",
+                "score": 1,
+                "user_id": user.username,
+            },
+        ],
+        "offset": 0,
+        "total_count": 0,
+    }
+    mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "get_user_feedback",
+        side_effect=[feedbacks, second_feedback],
+    )
+
+    funkwhale_ready.sync_favorites_from_listenbrainz(user, conf)
+
+    assert favorites_models.TrackFavorite.objects.filter(
+        track__mbid="195565db-65f9-4d0d-b347-5f0c85509528"
+    ).exists()
+    assert not favorites_models.TrackFavorite.objects.filter(
+        track__mbid="1fd02cf2-7247-4715-8862-c378ec196000"
+    ).exists()
+    with pytest.raises(favorites_models.TrackFavorite.DoesNotExist):
+        favorite.refresh_from_db()
+
+
+def test_submit_favorites_to_listenbrainz(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+
+    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+
+    favorite = factories["favorites.TrackFavorite"](track=track)
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "submit_favorites": True,
+    }
+
+    patch = mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "submit_user_feedback",
+        return_value="Success",
+    )
+    funkwhale_ready.submit_favorite_creation(favorite, conf)
+
+    patch.assert_called_once_with(1, track.mbid)
+
+
+def test_submit_favorites_deletion(factories, mocker, caplog):
+    logger = logging.getLogger("plugins")
+    caplog.set_level(logging.INFO)
+    logger.addHandler(caplog.handler)
+    user = factories["users.User"]()
+
+    factories["music.Track"](mbid="195565db-65f9-4d0d-b347-5f0c85509528")
+
+    factories["music.Track"](mbid="54c60860-f43d-484e-b691-7ab7ec8de559")
+    track = factories["music.Track"](mbid="c5af5351-dbbf-4481-b52e-a480b6c57986")
+
+    favorite = factories["favorites.TrackFavorite"](track=track)
+    conf = {
+        "user_name": user.username,
+        "user_token": "user_tolkien",
+        "submit_favorites": True,
+    }
+
+    patch = mocker.patch.object(
+        funkwhale_ready.tasks.liblistenbrainz.ListenBrainz,
+        "submit_user_feedback",
+        return_value="Success",
+    )
+    funkwhale_ready.submit_favorite_deletion(favorite, conf)
+
+    patch.assert_called_once_with(0, track.mbid)