diff --git a/django_kepi/models.py b/django_kepi/models.py index 94f74dd..8deedf0 100644 --- a/django_kepi/models.py +++ b/django_kepi/models.py @@ -29,9 +29,23 @@ class QuarantinedMessage(models.Model): default=False, ) - def deploy(self): + def deploy(self, + retrying=False): + + if retrying: + logger.debug('%s: re-attempting to deploy', self) + + remaining_qmns = QuarantinedMessageNeeds.objects.filter( + message=self.pk, + ) + + if remaining_qmns.exists(): + logger.debug('%s: -- but there are dependencies remaining: %s', + self, remaining_qmns) + return None + else: + logger.debug('%s: attempting to deploy', self) - logger.debug('%s: attempting to deploy', self) try: value = json.loads(self.body) except json.decoder.JSONDecodeError: @@ -45,17 +59,21 @@ class QuarantinedMessage(models.Model): local = False, ) except NeedToFetchException as ntfe: - logger.debug('%s: deployment failed because we need to fetch:', self) + logger.debug('%s: deployment failed because we need to fetch: %s', + self, ntfe.urls) + + if retrying: + logger.error("%s: dependencies remaining when all dependency records were gone; this should never happen") + raise RuntimeError("dependencies remaining on retry") + for need in ntfe.urls: qmn = QuarantinedMessageNeeds( message=self, needs_to_fetch=need, ) qmn.save() - logger.debug('%s: -- we need %s', self, need) qmn.start_looking() - logger.debug('%s: end of list', self) - + return None else: logger.info('%s: deployment was successful', self) diff --git a/django_kepi/views.py b/django_kepi/views.py index 34f979e..d9963ea 100644 --- a/django_kepi/views.py +++ b/django_kepi/views.py @@ -157,7 +157,7 @@ class InboxView(django.views.View): capture.save() try: - capture.deploy() + capture.deploy(retrying=False) except NeedToFetchException: # we'll work it out later pass @@ -219,31 +219,34 @@ class AsyncResultView(django.views.View): message_need.needs_to_fetch) logger.debug(' -- its contents are %s', body) else: - logger.debug('Batch processing has failed to retrieve %s:', + logger.info('Batch processing has failed to retrieve %s:', message_need.needs_to_fetch) if body is not None: try: fields = json.loads(body) + kepi_create(fields) except json.decoder.JSONDecodeError: fields = None success = False logger.warn('Body was not JSON. Treating as failure.') - kepi_create(json.loads(body)) + if success: + logger.debug(' -- trying to deploy all matching messages again') + else: + logger.debug(' -- deleting all messages which relied on it') - logger.debug(' -- trying to deploy all matching messages again') for need in list(QuarantinedMessageNeeds.objects.filter( needs_to_fetch = message_need.needs_to_fetch)): logger.debug(' -- %s', str(need.message)) if success: - need.message.deploy() + need.message.deploy(retrying=True) else: need.message.delete() need.delete() - logger.debug(' -- finished deployment attempts') + logger.debug(' -- finished') return HttpResponse( status = 200, diff --git a/tests/test_asyncresult.py b/tests/test_asyncresult.py index 656194c..e0f4b64 100644 --- a/tests/test_asyncresult.py +++ b/tests/test_asyncresult.py @@ -2,6 +2,9 @@ from django.test import TestCase, Client from django_kepi.models import Activity, QuarantinedMessage, QuarantinedMessageNeeds from django_kepi import create, resolve +# XXX bug is: +# XXX attempts to redeploy cause duplicate "needs" records + class TestAsyncResult(TestCase): def test_simple(self): @@ -55,7 +58,7 @@ class TestAsyncResult(TestCase): PERSON_URL = 'https://example.net/mary' ARTICLE_URL = 'https://example.com/articles/lambs-are-nice' QUARANTINED_BODY = """{ - "id": "https://example.com/id/1", + "id": "https://example.com/id/2", "type": "Like", "actor": "%s", "object": "%s" @@ -91,7 +94,7 @@ class TestAsyncResult(TestCase): need_person_uuid = QuarantinedMessageNeeds.objects.get(needs_to_fetch=PERSON_URL).id - c.post('/asyncResult?success=True&uuid=%s' % (need_person.id,), + c.post('/asyncResult?success=True&uuid=%s' % (need_person_uuid,), content_type = 'application/activity+json', data = { 'id': PERSON_URL, @@ -105,3 +108,59 @@ class TestAsyncResult(TestCase): QuarantinedMessageNeeds.objects.filter(needs_to_fetch=ARTICLE_URL).exists()) self.assertFalse( QuarantinedMessageNeeds.objects.filter(needs_to_fetch=PERSON_URL).exists()) + + # XXX assert that the activity now exists + + def test_failure(self): + + PERSON_URL = 'https://example.net/lucy' + ARTICLE_URL = 'https://example.com/articles/losing-your-pocket' + QUARANTINED_BODY = """{ + "id": "https://example.com/id/3", + "type": "Like", + "actor": "%s", + "object": "%s" + }""" % (PERSON_URL, ARTICLE_URL,) + + qlike = QuarantinedMessage( + username=None, + headers='', + body=QUARANTINED_BODY, + ) + qlike.save() + qlike.deploy() + + c = Client() + + need_article_uuid = QuarantinedMessageNeeds.objects.get(needs_to_fetch=ARTICLE_URL).id + + c.post('/asyncResult?success=True&uuid=%s' % (need_article_uuid,), + content_type = 'application/activity+json', + data = { + 'id': ARTICLE_URL, + "type": "Article", + "title": "Losing your pocket", + }, + ) + + self.assertTrue( + QuarantinedMessage.objects.filter(body=QUARANTINED_BODY).exists()) + self.assertFalse( + QuarantinedMessageNeeds.objects.filter(needs_to_fetch=ARTICLE_URL).exists()) + self.assertTrue( + QuarantinedMessageNeeds.objects.filter(needs_to_fetch=PERSON_URL).exists()) + + # But the person check fails! + need_person_uuid = QuarantinedMessageNeeds.objects.get(needs_to_fetch=PERSON_URL).id + + c.post('/asyncResult?success=False&uuid=%s' % (need_person_uuid,), + ) + + self.assertFalse( + QuarantinedMessage.objects.filter(body=QUARANTINED_BODY).exists()) + self.assertFalse( + QuarantinedMessageNeeds.objects.filter(needs_to_fetch=ARTICLE_URL).exists()) + self.assertFalse( + QuarantinedMessageNeeds.objects.filter(needs_to_fetch=PERSON_URL).exists()) + + # XXX assert that the activity does NOT exist