kopia lustrzana https://github.com/snarfed/bridgy-fed
Web: add poll feed task for new users, drop task if they've sent a webmention
Ryan Barrett
rodzic
28fa7eba8c
commit
e82555ad91
|
|
@ -615,11 +615,6 @@ class Object(StringIdModel):
|
|||
rel_urls=self.mf2.get('rel-urls'))
|
||||
use_urls_as_ids(obj)
|
||||
|
||||
# TODO: remove once we drop superfeedr
|
||||
elif self.atom:
|
||||
obj = atom.atom_to_activity(self.atom)['object']
|
||||
use_urls_as_ids(obj)
|
||||
|
||||
else:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Unit tests for webmention.py."""
|
||||
import copy
|
||||
from datetime import timedelta
|
||||
from unittest import skip
|
||||
from unittest.mock import ANY, patch
|
||||
|
||||
from flask import g, get_flashed_messages
|
||||
|
|
@ -513,63 +512,20 @@ class WebTest(TestCase):
|
|||
self.assert_entities_equal(user, Web.get_by_id('foo.bar'))
|
||||
self.assertIsNone(Web.get_by_id('..foo.bar.'))
|
||||
|
||||
@skip
|
||||
def test_get_or_create_subscribes_superfeedr(self, mock_get, mock_post):
|
||||
self.user.obj.mf2 = ACTOR_MF2_REL_FEED_URL
|
||||
self.user.obj.put()
|
||||
self.user.has_redirects = False
|
||||
self.user.put()
|
||||
|
||||
@patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task')
|
||||
def test_get_or_create_existing_no_poll_feed_task(self, mock_create_task, _, __):
|
||||
user = Web.get_or_create('user.com')
|
||||
self.assertEqual('user.com', user.key.id())
|
||||
self.assert_entities_equal(
|
||||
user, self.user,
|
||||
ignore=['superfeedr_subscribed', 'superfeedr_subscribed_feed', 'updated'])
|
||||
self.assertEqual(NOW, user.superfeedr_subscribed)
|
||||
self.assertEqual('https://foo/atom', user.superfeedr_subscribed_feed)
|
||||
mock_create_task.assert_not_called()
|
||||
|
||||
self.assert_req(mock_post, SUPERFEEDR_PUSH_API, data={
|
||||
'hub.mode': 'subscribe',
|
||||
'hub.topic': 'https://foo/atom',
|
||||
'hub.callback': 'http://localhost/superfeedr/notify/user.com',
|
||||
'format': 'atom',
|
||||
'retrieve': 'true',
|
||||
}, auth=ANY)
|
||||
self.assertEqual(NOW, self.user.key.get().superfeedr_subscribed)
|
||||
@patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task')
|
||||
def test_get_or_create_new_creates_poll_feed_task(self, mock_create_task,
|
||||
mock_get, __):
|
||||
common.RUN_TASKS_INLINE = False
|
||||
mock_get.return_value = ACTOR_HTML_RESP
|
||||
|
||||
def test_get_or_create_subscribe_error(self, mock_get, mock_post):
|
||||
self.user.obj.mf2 = ACTOR_MF2_REL_FEED_URL
|
||||
self.user.obj.put()
|
||||
self.user.has_redirects = False
|
||||
self.user.put()
|
||||
|
||||
mock_post.return_value = requests_response('Nope', status=500)
|
||||
|
||||
user = Web.get_or_create('user.com')
|
||||
self.assert_entities_equal(user, self.user, ignore=['updated'])
|
||||
self.assertIsNone(user.superfeedr_subscribed)
|
||||
self.assertIsNone(user.superfeedr_subscribed_feed)
|
||||
|
||||
def test_get_or_create_existing_subscribed(self, *_):
|
||||
self.user.superfeedr_subscribed = NOW
|
||||
self.user.put()
|
||||
|
||||
user = Web.get_or_create('user.com')
|
||||
self.assertEqual('user.com', user.key.id())
|
||||
self.assert_entities_equal(user, self.user)
|
||||
|
||||
def test_get_or_create_existing_has_last_webmention(self, *_):
|
||||
self.user.last_webmention_in = NOW
|
||||
self.user.put()
|
||||
|
||||
user = Web.get_or_create('user.com')
|
||||
self.assertEqual('user.com', user.key.id())
|
||||
self.assert_entities_equal(user, self.user)
|
||||
|
||||
def test_get_or_create_existing_not_subscribed_no_feed(self, *_):
|
||||
user = Web.get_or_create('user.com')
|
||||
self.assertEqual('user.com', user.key.id())
|
||||
self.assert_entities_equal(user, self.user)
|
||||
user = Web.get_or_create('new.com')
|
||||
self.assert_task(mock_create_task, 'poll-feed', '/queue/poll-feed',
|
||||
domain='new.com')
|
||||
|
||||
def test_get_or_create_existing_opted_out(self, *_):
|
||||
self.user.obj.mf2['properties']['summary'] = '#nobridge'
|
||||
|
|
@ -633,28 +589,6 @@ class WebTest(TestCase):
|
|||
|
||||
self.assertEqual(NOW, self.user.key.get().last_webmention_in)
|
||||
|
||||
@skip
|
||||
def test_first_webmention_unsubscribe_superfeedr(self, mock_get, mock_post):
|
||||
self.user.superfeedr_subscribed = NOW
|
||||
self.user.superfeedr_subscribed_feed = 'http://feed'
|
||||
self.user.put()
|
||||
|
||||
mock_get.return_value = NOTE
|
||||
|
||||
params = {
|
||||
'source': 'https://user.com/post',
|
||||
'target': 'https://fed.brid.gy/',
|
||||
}
|
||||
got = self.post('/webmention', data=params)
|
||||
self.assertEqual(204, got.status_code)
|
||||
|
||||
self.assertEqual(NOW, self.user.key.get().last_webmention_in)
|
||||
self.assert_req(mock_post, SUPERFEEDR_PUSH_API, data={
|
||||
'hub.mode': 'unsubscribe',
|
||||
'hub.topic': 'http://feed',
|
||||
'hub.callback': 'http://localhost/superfeedr/notify/user.com',
|
||||
}, auth=ANY)
|
||||
|
||||
def test_no_user(self, mock_get, mock_post):
|
||||
orig_count = Object.query().count()
|
||||
|
||||
|
|
@ -1996,6 +1930,26 @@ class WebTest(TestCase):
|
|||
self.assertEqual(200, got.status_code)
|
||||
self.assertIsNone(self.user.key.get().last_polled_feed)
|
||||
|
||||
@patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task')
|
||||
def test_poll_feed_last_webmention_in_noop(self, mock_create_task, mock_get, _):
|
||||
common.RUN_TASKS_INLINE = False
|
||||
|
||||
self.user.last_webmention_in = NOW
|
||||
self.user.put()
|
||||
self.user.obj.mf2 = {
|
||||
**ACTOR_MF2,
|
||||
'rel-urls': {
|
||||
'https://foo/rss': {'rels': ['alternate'], 'type': rss.CONTENT_TYPE},
|
||||
},
|
||||
}
|
||||
self.user.obj.put()
|
||||
|
||||
got = self.post('/queue/poll-feed', data={'domain': 'user.com'})
|
||||
self.assertEqual(200, got.status_code)
|
||||
self.assertIsNone(self.user.key.get().last_polled_feed)
|
||||
mock_create_task.assert_not_called()
|
||||
mock_get.assert_not_called()
|
||||
|
||||
def _test_verify(self, redirects, hcard, actor, redirects_error=None):
|
||||
self.user.has_redirects = False
|
||||
self.user.put()
|
||||
|
|
|
|||
52
web.py
52
web.py
|
|
@ -100,8 +100,6 @@ class Web(User, Protocol):
|
|||
has_hcard = ndb.BooleanProperty()
|
||||
last_webmention_in = ndb.DateTimeProperty(tzinfo=timezone.utc)
|
||||
last_polled_feed = ndb.DateTimeProperty(tzinfo=timezone.utc)
|
||||
superfeedr_subscribed = ndb.DateTimeProperty(tzinfo=timezone.utc)
|
||||
superfeedr_subscribed_feed = ndb.StringProperty()
|
||||
|
||||
# Originally, BF served Web users' AP actor ids on fed.brid.gy, eg
|
||||
# https://fed.brid.gy/snarfed.org . When we started adding new protocols, we
|
||||
|
|
@ -110,6 +108,10 @@ class Web(User, Protocol):
|
|||
# property tracks which subdomain a given Web user's AP actor uses.
|
||||
ap_subdomain = ndb.StringProperty(choices=['fed', 'web'], default='web')
|
||||
|
||||
# OLD. some stored entities still have these; do not reuse.
|
||||
# superfeedr_subscribed = ndb.DateTimeProperty(tzinfo=timezone.utc)
|
||||
# superfeedr_subscribed_feed = ndb.StringProperty()
|
||||
|
||||
@classmethod
|
||||
def _get_kind(cls):
|
||||
return 'MagicKey'
|
||||
|
|
@ -123,7 +125,7 @@ class Web(User, Protocol):
|
|||
|
||||
@classmethod
|
||||
def get_or_create(cls, id, **kwargs):
|
||||
"""Normalize domain, pass through, then subscribe in Superfeedr.
|
||||
"""Normalize domain, then pass through to :meth:`User.get_or_create`.
|
||||
|
||||
Normalizing currently consists of lower casing and removing leading and
|
||||
trailing dots.
|
||||
|
|
@ -135,8 +137,8 @@ class Web(User, Protocol):
|
|||
domain = key.id().lower().strip('.')
|
||||
user = super().get_or_create(domain, **kwargs)
|
||||
|
||||
# TODO
|
||||
# maybe_superfeedr_subscribe(user)
|
||||
if not user.existing:
|
||||
common.create_task(queue='poll-feed', domain=domain)
|
||||
|
||||
return user
|
||||
|
||||
|
|
@ -586,8 +588,6 @@ def webmention_external():
|
|||
if request.path == '/webmention': # exclude interactive
|
||||
user.last_webmention_in = util.now()
|
||||
user.put()
|
||||
# TODO
|
||||
# maybe_superfeedr_unsubscribe(user)
|
||||
|
||||
return common.create_task('webmention', **request.form)
|
||||
|
||||
|
|
@ -611,23 +611,6 @@ def webmention_interactive():
|
|||
return redirect('/', code=302)
|
||||
|
||||
|
||||
def maybe_superfeedr_subscribe(user):
|
||||
"""Subscribes to a user's Atom or RSS feed in Superfeedr.
|
||||
|
||||
Args:
|
||||
user (Web)
|
||||
"""
|
||||
if user.superfeedr_subscribed:
|
||||
logger.info(f'User {user.key.id()} already subscribed via Superfeedr')
|
||||
return
|
||||
elif user.has_redirects or user.last_webmention_in:
|
||||
logger.info(f'User {user.key.id()} has Webfinger redirects or publishes via webmention, not subscribing via Superfeedr')
|
||||
return
|
||||
elif not user.obj or not user.obj.mf2:
|
||||
logger.info(f"User {user.key.id()} has no mf2, can't subscribe via Superfeedr")
|
||||
return
|
||||
|
||||
|
||||
@app.post(f'/queue/poll-feed')
|
||||
def poll_feed_task():
|
||||
"""Fetches a :class:`Web` site's feed and delivers new/updated posts.
|
||||
|
|
@ -635,9 +618,13 @@ def poll_feed_task():
|
|||
Params:
|
||||
``domain`` (str): key id of the :class:`Web` user
|
||||
"""
|
||||
user = Web.get_by_id(flask_util.get_required_param('domain'))
|
||||
if not user:
|
||||
error(f'No Web user found for domain {domain}', status=304)
|
||||
domain = flask_util.get_required_param('domain')
|
||||
user = Web.get_by_id(domain)
|
||||
if not (user and user.obj and user.obj.mf2):
|
||||
error(f'No Web user or object found for domain {domain}', status=304)
|
||||
elif user.last_webmention_in:
|
||||
logger.info(f'Dropping since last_webmention_in is set')
|
||||
return 'OK'
|
||||
|
||||
# discover feed URL
|
||||
for url, info in user.obj.mf2.get('rel-urls', {}).items():
|
||||
|
|
@ -692,13 +679,14 @@ def poll_feed_task():
|
|||
authed_as=user.key.id())
|
||||
|
||||
# create next poll task
|
||||
def clamp(delay):
|
||||
return max(min(delay, MAX_FEED_POLL_PERIOD), MIN_FEED_POLL_PERIOD)
|
||||
|
||||
if published_deltas:
|
||||
seconds = statistics.mean(t.total_seconds() for t in published_deltas)
|
||||
delay = max(min(timedelta(seconds=seconds), MAX_FEED_POLL_PERIOD),
|
||||
MIN_FEED_POLL_PERIOD)
|
||||
delay = clamp(timedelta(seconds=statistics.mean(
|
||||
t.total_seconds() for t in published_deltas)))
|
||||
else:
|
||||
# TODO
|
||||
delay = MIN_FEED_POLL_PERIOD
|
||||
delay = clamp(util.now() - (user.last_polled_feed or user.created))
|
||||
|
||||
common.create_task(queue='poll-feed', domain=user.key.id(), delay=delay)
|
||||
return 'OK'
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue