intermediate

django_kepi/models.py

@@ -29,9 +29,23 @@ class QuarantinedMessage(models.Model):
             default=False,
             )
 
-    def deploy(self):
+    def deploy(self,
+            retrying=False):
+
+        if retrying:
+            logger.debug('%s: re-attempting to deploy', self)
+
+            remaining_qmns = QuarantinedMessageNeeds.objects.filter(
+                    message=self.pk,
+                    )
+            
+            if remaining_qmns.exists():
+                logger.debug('%s:  -- but there are dependencies remaining: %s',
+                        self, remaining_qmns)
+                return None
+        else:
+            logger.debug('%s: attempting to deploy', self)
 
-        logger.debug('%s: attempting to deploy', self)
         try:
             value = json.loads(self.body)
         except json.decoder.JSONDecodeError:
@@ -45,17 +59,21 @@ class QuarantinedMessage(models.Model):
                     local = False,
                     )
         except NeedToFetchException as ntfe:
-            logger.debug('%s: deployment failed because we need to fetch:', self)
+            logger.debug('%s: deployment failed because we need to fetch: %s',
+                    self, ntfe.urls)
+
+            if retrying:
+                logger.error("%s: dependencies remaining when all dependency records were gone; this should never happen")
+                raise RuntimeError("dependencies remaining on retry")
+
             for need in ntfe.urls:
                 qmn = QuarantinedMessageNeeds(
                         message=self,
                         needs_to_fetch=need,
                         )
                 qmn.save()
-                logger.debug('%s:   -- we need %s', self, need)
                 qmn.start_looking()
-            logger.debug('%s: end of list', self)
-
+                
             return None
         else:
             logger.info('%s: deployment was successful', self)

django_kepi/views.py

@@ -157,7 +157,7 @@ class InboxView(django.views.View):
         capture.save()
 
         try:
-            capture.deploy()
+            capture.deploy(retrying=False)
         except NeedToFetchException:
             # we'll work it out later
             pass
@@ -219,31 +219,34 @@ class AsyncResultView(django.views.View):
                     message_need.needs_to_fetch)
             logger.debug(' -- its contents are %s', body)
         else:
-            logger.debug('Batch processing has failed to retrieve %s:',
+            logger.info('Batch processing has failed to retrieve %s:',
                     message_need.needs_to_fetch)
 
         if body is not None:
             try:
                 fields = json.loads(body)
+                kepi_create(fields)
             except json.decoder.JSONDecodeError:
                 fields = None
                 success = False
                 logger.warn('Body was not JSON. Treating as failure.')
 
-        kepi_create(json.loads(body))
+        if success:
+            logger.debug(' -- trying to deploy all matching messages again')
+        else:
+            logger.debug(' -- deleting all messages which relied on it')
 
-        logger.debug(' -- trying to deploy all matching messages again')
         for need in list(QuarantinedMessageNeeds.objects.filter(
             needs_to_fetch = message_need.needs_to_fetch)):
 
             logger.debug('    -- %s', str(need.message))
             if success:
-                need.message.deploy()
+                need.message.deploy(retrying=True)
             else:
                 need.message.delete()
 
             need.delete()
-        logger.debug(' -- finished deployment attempts')
+        logger.debug(' -- finished')
 
         return HttpResponse(
                 status = 200,

tests/test_asyncresult.py

@@ -2,6 +2,9 @@ from django.test import TestCase, Client
 from django_kepi.models import Activity, QuarantinedMessage, QuarantinedMessageNeeds
 from django_kepi import create, resolve
 
+# XXX bug is:
+# XXX   attempts to redeploy cause duplicate "needs" records
+
 class TestAsyncResult(TestCase):
 
     def test_simple(self):
@@ -55,7 +58,7 @@ class TestAsyncResult(TestCase):
         PERSON_URL = 'https://example.net/mary'
         ARTICLE_URL = 'https://example.com/articles/lambs-are-nice'
         QUARANTINED_BODY = """{
-            "id": "https://example.com/id/1",
+            "id": "https://example.com/id/2",
             "type": "Like",
             "actor": "%s",
             "object": "%s"
@@ -91,7 +94,7 @@ class TestAsyncResult(TestCase):
 
         need_person_uuid = QuarantinedMessageNeeds.objects.get(needs_to_fetch=PERSON_URL).id
 
-        c.post('/asyncResult?success=True&uuid=%s' % (need_person.id,),
+        c.post('/asyncResult?success=True&uuid=%s' % (need_person_uuid,),
                 content_type = 'application/activity+json',
                 data = {
                     'id': PERSON_URL,
@@ -105,3 +108,59 @@ class TestAsyncResult(TestCase):
                 QuarantinedMessageNeeds.objects.filter(needs_to_fetch=ARTICLE_URL).exists())
         self.assertFalse(
                 QuarantinedMessageNeeds.objects.filter(needs_to_fetch=PERSON_URL).exists())
+
+        # XXX assert that the activity now exists
+
+    def test_failure(self):
+        
+        PERSON_URL = 'https://example.net/lucy'
+        ARTICLE_URL = 'https://example.com/articles/losing-your-pocket'
+        QUARANTINED_BODY = """{
+            "id": "https://example.com/id/3",
+            "type": "Like",
+            "actor": "%s",
+            "object": "%s"
+            }""" % (PERSON_URL, ARTICLE_URL,)
+
+        qlike = QuarantinedMessage(
+                username=None,
+                headers='',
+                body=QUARANTINED_BODY,
+            )
+        qlike.save()
+        qlike.deploy()
+
+        c = Client()
+
+        need_article_uuid = QuarantinedMessageNeeds.objects.get(needs_to_fetch=ARTICLE_URL).id
+
+        c.post('/asyncResult?success=True&uuid=%s' % (need_article_uuid,),
+                content_type = 'application/activity+json',
+                data = {
+                    'id': ARTICLE_URL,
+                    "type": "Article",
+                    "title": "Losing your pocket",
+                    },
+                )
+
+        self.assertTrue(
+                QuarantinedMessage.objects.filter(body=QUARANTINED_BODY).exists())
+        self.assertFalse(
+                QuarantinedMessageNeeds.objects.filter(needs_to_fetch=ARTICLE_URL).exists())
+        self.assertTrue(
+                QuarantinedMessageNeeds.objects.filter(needs_to_fetch=PERSON_URL).exists())
+
+        # But the person check fails!
+        need_person_uuid = QuarantinedMessageNeeds.objects.get(needs_to_fetch=PERSON_URL).id
+
+        c.post('/asyncResult?success=False&uuid=%s' % (need_person_uuid,),
+                )
+
+        self.assertFalse(
+                QuarantinedMessage.objects.filter(body=QUARANTINED_BODY).exists())
+        self.assertFalse(
+                QuarantinedMessageNeeds.objects.filter(needs_to_fetch=ARTICLE_URL).exists())
+        self.assertFalse(
+                QuarantinedMessageNeeds.objects.filter(needs_to_fetch=PERSON_URL).exists())
+
+        # XXX assert that the activity does NOT exist