2021-07-28 04:58:05 +00:00
|
|
|
"""Datastore model classes."""
|
2023-04-18 16:08:45 +00:00
|
|
|
from datetime import timedelta, timezone
|
2023-03-08 21:10:41 +00:00
|
|
|
import itertools
|
2023-04-28 19:02:26 +00:00
|
|
|
import json
|
2017-10-26 14:31:15 +00:00
|
|
|
import logging
|
2023-04-25 21:04:29 +00:00
|
|
|
import random
|
2019-12-26 06:20:57 +00:00
|
|
|
import urllib.parse
|
2017-10-11 05:42:19 +00:00
|
|
|
|
2023-05-06 21:37:23 +00:00
|
|
|
from arroba.mst import dag_cbor_cid
|
2023-01-05 03:22:11 +00:00
|
|
|
from Crypto import Random
|
2023-04-25 21:04:29 +00:00
|
|
|
from Crypto.PublicKey import ECC, RSA
|
2023-04-28 19:02:26 +00:00
|
|
|
import dag_json
|
2023-03-20 21:28:14 +00:00
|
|
|
from flask import g, request
|
2019-12-26 06:20:57 +00:00
|
|
|
from google.cloud import ndb
|
2023-02-24 03:17:26 +00:00
|
|
|
from granary import as1, as2, bluesky, microformats2
|
2023-05-24 13:49:54 +00:00
|
|
|
from oauth_dropins.webutil import util
|
2023-02-06 06:22:22 +00:00
|
|
|
from oauth_dropins.webutil.appengine_info import DEBUG
|
2023-05-24 13:49:54 +00:00
|
|
|
from oauth_dropins.webutil.flask_util import error
|
2023-02-24 14:25:18 +00:00
|
|
|
from oauth_dropins.webutil.models import ComputedJsonProperty, JsonProperty, StringIdModel
|
2022-11-20 17:38:46 +00:00
|
|
|
from oauth_dropins.webutil.util import json_dumps, json_loads
|
2023-04-28 19:02:26 +00:00
|
|
|
import requests
|
2017-08-19 15:36:55 +00:00
|
|
|
|
2022-11-19 02:46:27 +00:00
|
|
|
import common
|
2023-05-31 17:47:09 +00:00
|
|
|
from common import base64_to_long, long_to_base64, redirect_unwrap, redirect_wrap
|
2022-11-19 02:46:27 +00:00
|
|
|
|
2023-05-26 23:07:36 +00:00
|
|
|
# maps string label to Protocol subclass. populated by ProtocolUserMeta.
|
|
|
|
# seed with old and upcoming protocols that don't have their own classes (yet).
|
|
|
|
PROTOCOLS = {'bluesky': None, 'ostatus': None}
|
|
|
|
|
2023-02-06 06:22:22 +00:00
|
|
|
# 2048 bits makes tests slow, so use 1024 for them
|
|
|
|
KEY_BITS = 1024 if DEBUG else 2048
|
2023-03-08 21:10:41 +00:00
|
|
|
PAGE_SIZE = 20
|
2023-01-05 03:22:11 +00:00
|
|
|
|
2023-04-18 16:08:45 +00:00
|
|
|
# auto delete old objects of these types via the Object.expire property
|
|
|
|
# https://cloud.google.com/datastore/docs/ttl
|
|
|
|
OBJECT_EXPIRE_TYPES = (
|
|
|
|
'post',
|
|
|
|
'update',
|
|
|
|
'delete',
|
|
|
|
'accept',
|
|
|
|
'reject',
|
|
|
|
'undo',
|
|
|
|
None
|
|
|
|
)
|
|
|
|
OBJECT_EXPIRE_AGE = timedelta(days=90)
|
|
|
|
|
2022-02-12 06:38:56 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
2017-08-19 15:36:55 +00:00
|
|
|
|
2023-05-26 23:07:36 +00:00
|
|
|
class ProtocolUserMeta(type(ndb.Model)):
|
|
|
|
""":class:`User` metaclass. Registers all subclasses in the PROTOCOLS global."""
|
|
|
|
def __new__(meta, name, bases, class_dict):
|
|
|
|
cls = super().__new__(meta, name, bases, class_dict)
|
|
|
|
if hasattr(cls, 'LABEL'):
|
|
|
|
PROTOCOLS[cls.LABEL] = cls
|
|
|
|
return cls
|
|
|
|
|
|
|
|
|
|
|
|
def reset_protocol_properties():
|
|
|
|
"""Recreates various protocol properties to include choices PROTOCOLS."""
|
2023-05-26 23:36:45 +00:00
|
|
|
Target.protocol = ndb.StringProperty(
|
|
|
|
'protocol', choices=list(PROTOCOLS.keys()), required=True)
|
|
|
|
Object.source_protocol = ndb.StringProperty(
|
|
|
|
'source_protocol', choices=list(PROTOCOLS.keys()))
|
2023-05-26 23:07:36 +00:00
|
|
|
|
|
|
|
|
|
|
|
class User(StringIdModel, metaclass=ProtocolUserMeta):
|
|
|
|
"""Abstract base class for a Bridgy Fed user.
|
2017-08-19 16:15:29 +00:00
|
|
|
|
2023-04-25 21:04:29 +00:00
|
|
|
Stores multiple keypairs needed for the supported protocols. Currently:
|
|
|
|
|
|
|
|
* RSA keypair for ActivityPub HTTP Signatures
|
2023-05-26 23:07:36 +00:00
|
|
|
properties: mod, public_exponent, private_exponent, all encoded as
|
|
|
|
base64url (ie URL-safe base64) strings as described in RFC 4648 and
|
|
|
|
section 5.1 of the Magic Signatures spec
|
2023-04-25 21:04:29 +00:00
|
|
|
https://tools.ietf.org/html/draft-cavage-http-signatures-12
|
|
|
|
|
|
|
|
* P-256 keypair for AT Protocol's signing key
|
|
|
|
property: p256_key, PEM encoded
|
|
|
|
https://atproto.com/guides/overview#account-portability
|
2017-08-19 16:15:29 +00:00
|
|
|
"""
|
2023-05-31 20:17:17 +00:00
|
|
|
mod = ndb.StringProperty()
|
|
|
|
public_exponent = ndb.StringProperty()
|
|
|
|
private_exponent = ndb.StringProperty()
|
2023-04-25 21:04:29 +00:00
|
|
|
p256_key = ndb.StringProperty()
|
2022-11-19 02:46:27 +00:00
|
|
|
has_redirects = ndb.BooleanProperty()
|
2022-11-28 01:33:53 +00:00
|
|
|
redirects_error = ndb.TextProperty()
|
2022-11-19 02:46:27 +00:00
|
|
|
has_hcard = ndb.BooleanProperty()
|
2023-02-24 13:25:29 +00:00
|
|
|
actor_as2 = JsonProperty()
|
2022-12-02 18:48:16 +00:00
|
|
|
use_instead = ndb.KeyProperty()
|
2017-08-19 16:15:29 +00:00
|
|
|
|
2023-05-30 02:37:35 +00:00
|
|
|
# whether this user signed up or otherwise explicitly, deliberately
|
|
|
|
# interacted with Bridgy Fed. For example, if fediverse user @a@b.com looks
|
|
|
|
# up @foo.com@fed.brid.gy via WebFinger, we'll create Users for both,
|
|
|
|
# @a@b.com will be direct, foo.com will not.
|
|
|
|
direct = ndb.BooleanProperty(default=False)
|
|
|
|
|
2022-12-02 22:46:09 +00:00
|
|
|
created = ndb.DateTimeProperty(auto_now_add=True)
|
|
|
|
updated = ndb.DateTimeProperty(auto_now=True)
|
|
|
|
|
2023-06-04 04:48:15 +00:00
|
|
|
@ndb.ComputedProperty
|
|
|
|
def readable_id(self):
|
|
|
|
"""This user's human-readable unique id, eg '@me@snarfed.org'.
|
|
|
|
|
|
|
|
To be implemented by subclasses.
|
|
|
|
"""
|
|
|
|
return None
|
|
|
|
|
|
|
|
def readable_or_key_id(self):
|
|
|
|
"""Returns readable_id if set, otherwise key id."""
|
|
|
|
return self.readable_id or self.key.id()
|
|
|
|
|
2023-05-26 23:07:36 +00:00
|
|
|
@classmethod
|
|
|
|
def new(cls, **kwargs):
|
|
|
|
"""Try to prevent instantiation. Use subclasses instead."""
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
2023-02-14 23:38:42 +00:00
|
|
|
def _post_put_hook(self, future):
|
2023-05-26 23:07:36 +00:00
|
|
|
logger.info(f'Wrote {self.key}')
|
2023-02-14 23:38:42 +00:00
|
|
|
|
2023-02-05 03:11:06 +00:00
|
|
|
@classmethod
|
|
|
|
def get_by_id(cls, id):
|
|
|
|
"""Override Model.get_by_id to follow the use_instead property."""
|
|
|
|
user = cls._get_by_id(id)
|
|
|
|
if user and user.use_instead:
|
|
|
|
return user.use_instead.get()
|
|
|
|
|
|
|
|
return user
|
|
|
|
|
2023-05-26 23:07:36 +00:00
|
|
|
@classmethod
|
2019-12-26 06:20:57 +00:00
|
|
|
@ndb.transactional()
|
2023-05-31 19:38:00 +00:00
|
|
|
def get_or_create(cls, id, **kwargs):
|
2022-11-16 06:00:28 +00:00
|
|
|
"""Loads and returns a User. Creates it if necessary."""
|
2023-05-26 23:07:36 +00:00
|
|
|
assert cls != User
|
2023-05-31 19:38:00 +00:00
|
|
|
user = cls.get_by_id(id)
|
2023-04-25 21:04:29 +00:00
|
|
|
if user:
|
2023-06-03 14:28:01 +00:00
|
|
|
# override direct from False => True if set
|
2023-05-30 02:37:35 +00:00
|
|
|
direct = kwargs.get('direct')
|
2023-06-02 19:55:07 +00:00
|
|
|
if direct and not user.direct:
|
2023-05-31 20:17:17 +00:00
|
|
|
logger.info(f'Setting {user.key} direct={direct}')
|
2023-05-30 02:37:35 +00:00
|
|
|
user.direct = direct
|
|
|
|
user.put()
|
2023-04-25 21:04:29 +00:00
|
|
|
return user
|
|
|
|
|
2023-05-26 23:07:36 +00:00
|
|
|
# generate keys for all protocols _except_ our own
|
|
|
|
#
|
|
|
|
# these can use urandom() and do nontrivial math, so they can take time
|
|
|
|
# depending on the amount of randomness available and compute needed.
|
|
|
|
if cls.LABEL != 'activitypub':
|
|
|
|
# originally from django_salmon.magicsigs
|
|
|
|
key = RSA.generate(KEY_BITS, randfunc=random.randbytes if DEBUG else None)
|
|
|
|
kwargs.update({
|
|
|
|
'mod': long_to_base64(key.n),
|
|
|
|
'public_exponent': long_to_base64(key.e),
|
|
|
|
'private_exponent': long_to_base64(key.d),
|
|
|
|
})
|
|
|
|
|
|
|
|
if cls.LABEL != 'atprotocol':
|
|
|
|
key = ECC.generate(
|
|
|
|
curve='P-256', randfunc=random.randbytes if DEBUG else None)
|
|
|
|
kwargs['p256_key'] = key.export_key(format='PEM')
|
|
|
|
|
2023-05-31 19:38:00 +00:00
|
|
|
user = cls(id=id, **kwargs)
|
2023-05-31 20:17:17 +00:00
|
|
|
logger.info(f'Created new {user}')
|
2023-04-25 21:04:29 +00:00
|
|
|
user.put()
|
2022-11-16 06:00:28 +00:00
|
|
|
return user
|
2017-08-19 16:15:29 +00:00
|
|
|
|
|
|
|
def href(self):
|
2023-01-24 20:17:24 +00:00
|
|
|
return f'data:application/magic-public-key,RSA.{self.mod}.{self.public_exponent}'
|
2017-09-19 14:13:35 +00:00
|
|
|
|
|
|
|
def public_pem(self):
|
2019-12-26 06:20:57 +00:00
|
|
|
"""Returns: bytes"""
|
2023-01-05 03:22:11 +00:00
|
|
|
rsa = RSA.construct((base64_to_long(str(self.mod)),
|
|
|
|
base64_to_long(str(self.public_exponent))))
|
2017-09-19 14:13:35 +00:00
|
|
|
return rsa.exportKey(format='PEM')
|
|
|
|
|
|
|
|
def private_pem(self):
|
2019-12-26 06:20:57 +00:00
|
|
|
"""Returns: bytes"""
|
2023-01-05 03:22:11 +00:00
|
|
|
rsa = RSA.construct((base64_to_long(str(self.mod)),
|
|
|
|
base64_to_long(str(self.public_exponent)),
|
|
|
|
base64_to_long(str(self.private_exponent))))
|
2017-09-19 14:13:35 +00:00
|
|
|
return rsa.exportKey(format='PEM')
|
2017-10-10 00:29:50 +00:00
|
|
|
|
2023-01-13 19:40:52 +00:00
|
|
|
def to_as1(self):
|
|
|
|
"""Returns this user as an AS1 actor dict, if possible."""
|
|
|
|
if self.actor_as2:
|
2023-02-24 13:25:29 +00:00
|
|
|
return as2.to_as1(self.actor_as2)
|
2023-01-13 19:40:52 +00:00
|
|
|
|
2023-06-02 04:37:58 +00:00
|
|
|
def name(self):
|
|
|
|
"""Returns this user's human-readable name, eg 'Ryan Barrett'."""
|
|
|
|
if self.actor_as2:
|
|
|
|
name = self.actor_as2.get('name')
|
|
|
|
if name:
|
|
|
|
return name
|
|
|
|
|
2023-06-04 04:48:15 +00:00
|
|
|
return self.readable_or_key_id()
|
2023-06-02 04:37:58 +00:00
|
|
|
|
2022-11-27 00:05:02 +00:00
|
|
|
def username(self):
|
2023-02-12 20:03:27 +00:00
|
|
|
"""Returns the user's preferred username.
|
2022-11-27 00:05:02 +00:00
|
|
|
|
2023-05-31 19:38:00 +00:00
|
|
|
Uses stored representative h-card if available, falls back to id.
|
2022-11-27 00:05:02 +00:00
|
|
|
|
2023-06-02 04:37:58 +00:00
|
|
|
TODO(#512): move to Web
|
|
|
|
|
2022-11-27 00:05:02 +00:00
|
|
|
Returns: str
|
|
|
|
"""
|
2023-05-31 19:38:00 +00:00
|
|
|
id = self.key.id()
|
2022-11-20 17:38:46 +00:00
|
|
|
|
2023-05-30 03:16:15 +00:00
|
|
|
if self.actor_as2 and self.direct:
|
2022-11-27 00:05:02 +00:00
|
|
|
for url in [u.get('value') if isinstance(u, dict) else u
|
2023-02-24 13:25:29 +00:00
|
|
|
for u in util.get_list(self.actor_as2, 'url')]:
|
2022-11-27 00:05:02 +00:00
|
|
|
if url and url.startswith('acct:'):
|
|
|
|
urluser, urldomain = util.parse_acct_uri(url)
|
2023-05-31 19:38:00 +00:00
|
|
|
if urldomain == id:
|
2022-11-27 00:05:02 +00:00
|
|
|
logger.info(f'Found custom username: {urluser}')
|
|
|
|
return urluser
|
|
|
|
|
2023-05-31 19:38:00 +00:00
|
|
|
logger.info(f'Defaulting username to key id {id}')
|
|
|
|
return id
|
2022-11-20 17:38:46 +00:00
|
|
|
|
2023-06-01 01:34:33 +00:00
|
|
|
def web_url(self):
|
|
|
|
"""Returns this user's web URL aka web_url, eg 'https://foo.com/'.
|
|
|
|
|
|
|
|
To be implemented by subclasses.
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
str
|
|
|
|
"""
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
def is_web_url(self, url):
|
|
|
|
"""Returns True if the given URL is this user's web URL (web_url).
|
|
|
|
|
|
|
|
Args:
|
|
|
|
url: str
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
boolean
|
|
|
|
"""
|
|
|
|
if not url:
|
|
|
|
return False
|
|
|
|
|
|
|
|
url = url.strip().rstrip('/')
|
|
|
|
parsed_url = urllib.parse.urlparse(url)
|
|
|
|
if parsed_url.scheme not in ('http', 'https', ''):
|
|
|
|
return False
|
|
|
|
|
|
|
|
this = self.web_url().rstrip('/')
|
|
|
|
parsed_this = urllib.parse.urlparse(this)
|
|
|
|
|
|
|
|
return (url == this or url == parsed_this.netloc or
|
|
|
|
parsed_url[1:] == parsed_this[1:]) # ignore http vs https
|
|
|
|
|
2023-05-31 17:10:14 +00:00
|
|
|
def ap_address(self):
|
2023-05-31 17:47:09 +00:00
|
|
|
"""Returns this user's ActivityPub address, eg '@me@foo.com'.
|
|
|
|
|
|
|
|
To be implemented by subclasses.
|
2023-06-01 01:34:33 +00:00
|
|
|
|
|
|
|
Returns:
|
|
|
|
str
|
2023-05-31 17:47:09 +00:00
|
|
|
"""
|
|
|
|
raise NotImplementedError()
|
2023-05-31 17:10:14 +00:00
|
|
|
|
|
|
|
def ap_actor(self, rest=None):
|
2023-05-31 17:47:09 +00:00
|
|
|
"""Returns this user's ActivityPub/AS2 actor id.
|
2023-05-31 17:10:14 +00:00
|
|
|
|
2023-05-31 17:47:09 +00:00
|
|
|
Eg 'https://fed.brid.gy/ap/bluesky/foo.com'
|
|
|
|
|
|
|
|
To be implemented by subclasses.
|
2023-06-01 01:34:33 +00:00
|
|
|
|
|
|
|
Args:
|
|
|
|
rest: str, optional, appended to URL path
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
str
|
2023-05-31 17:10:14 +00:00
|
|
|
"""
|
2023-05-31 17:47:09 +00:00
|
|
|
raise NotImplementedError()
|
2023-02-12 20:03:27 +00:00
|
|
|
|
2023-05-30 21:08:13 +00:00
|
|
|
def user_page_path(self, rest=None):
|
|
|
|
"""Returns the user's Bridgy Fed user page path."""
|
2023-06-04 04:48:15 +00:00
|
|
|
path = f'/{self.LABEL}/{self.readable_or_key_id()}'
|
2023-06-03 15:03:38 +00:00
|
|
|
|
2023-05-30 21:08:13 +00:00
|
|
|
if rest:
|
2023-06-03 15:03:38 +00:00
|
|
|
if not rest.startswith('?'):
|
|
|
|
path += '/'
|
|
|
|
path += rest
|
|
|
|
|
2023-05-30 21:08:13 +00:00
|
|
|
return path
|
|
|
|
|
2022-11-27 00:29:48 +00:00
|
|
|
def user_page_link(self):
|
|
|
|
"""Returns a pretty user page link with the user's name and profile picture."""
|
2023-02-24 13:25:29 +00:00
|
|
|
actor = self.actor_as2 or {}
|
2022-12-01 05:04:22 +00:00
|
|
|
img = util.get_url(actor, 'icon') or ''
|
2023-06-02 04:37:58 +00:00
|
|
|
return f'<a class="h-card u-author" href="{self.user_page_path()}"><img src="{img}" class="profile"> {self.name()}</a>'
|
2022-11-26 06:21:50 +00:00
|
|
|
|
2017-10-10 00:29:50 +00:00
|
|
|
|
2023-02-01 21:19:41 +00:00
|
|
|
class Target(ndb.Model):
|
|
|
|
"""Delivery destinations. ActivityPub inboxes, webmention targets, etc.
|
|
|
|
|
|
|
|
Used in StructuredPropertys inside Object; not stored directly in the
|
|
|
|
datastore.
|
|
|
|
|
|
|
|
ndb implements this by hoisting each property here into a corresponding
|
|
|
|
property on the parent entity, prefixed by the StructuredProperty name
|
|
|
|
below, eg delivered.uri, delivered.protocol, etc.
|
|
|
|
|
|
|
|
For repeated StructuredPropertys, the hoisted properties are all
|
|
|
|
repeated on the parent entity, and reconstructed into
|
|
|
|
StructuredPropertys based on their order.
|
|
|
|
|
|
|
|
https://googleapis.dev/python/python-ndb/latest/model.html#google.cloud.ndb.model.StructuredProperty
|
|
|
|
"""
|
|
|
|
uri = ndb.StringProperty(required=True)
|
2023-05-26 23:07:36 +00:00
|
|
|
# choices is populated in flask_app, after all User subclasses are created,
|
|
|
|
# so that PROTOCOLS is fully populated
|
|
|
|
protocol = ndb.StringProperty(choices=[], required=True)
|
2023-02-01 21:19:41 +00:00
|
|
|
|
|
|
|
|
2023-01-28 15:48:50 +00:00
|
|
|
class Object(StringIdModel):
|
|
|
|
"""An activity or other object, eg actor.
|
2017-10-10 00:29:50 +00:00
|
|
|
|
2023-01-28 15:48:50 +00:00
|
|
|
Key name is the id. We synthesize ids if necessary.
|
2017-10-10 00:29:50 +00:00
|
|
|
"""
|
2023-01-29 04:49:20 +00:00
|
|
|
STATUSES = ('new', 'in progress', 'complete', 'failed', 'ignored')
|
2023-02-01 20:22:04 +00:00
|
|
|
LABELS = ('activity', 'feed', 'notification', 'user')
|
2017-10-10 00:29:50 +00:00
|
|
|
|
2022-11-17 02:53:49 +00:00
|
|
|
# domains of the Bridgy Fed users this activity is to or from
|
2023-01-28 15:48:50 +00:00
|
|
|
domains = ndb.StringProperty(repeated=True)
|
2023-02-03 03:51:55 +00:00
|
|
|
status = ndb.StringProperty(choices=STATUSES)
|
2023-05-26 23:07:36 +00:00
|
|
|
# choices is populated in flask_app, after all User subclasses are created,
|
|
|
|
# so that PROTOCOLS is fully populated
|
2023-03-08 21:10:41 +00:00
|
|
|
# TODO: remove? is this redundant with the protocol-specific data fields below?
|
2023-05-26 23:07:36 +00:00
|
|
|
source_protocol = ndb.StringProperty(choices=[])
|
2023-01-28 15:48:50 +00:00
|
|
|
labels = ndb.StringProperty(repeated=True, choices=LABELS)
|
|
|
|
|
2023-02-24 13:25:29 +00:00
|
|
|
# TODO: switch back to ndb.JsonProperty if/when they fix it for the web console
|
|
|
|
# https://github.com/googleapis/python-ndb/issues/874
|
2023-03-21 16:10:14 +00:00
|
|
|
as2 = JsonProperty() # only one of the rest will be populated...
|
|
|
|
bsky = JsonProperty() # Bluesky / AT Protocol
|
2023-03-22 19:27:40 +00:00
|
|
|
mf2 = JsonProperty() # HTML microformats2 item (ie _not_ the top level
|
|
|
|
# parse object with items inside an 'items' field)
|
2023-03-21 16:10:14 +00:00
|
|
|
our_as1 = JsonProperty() # AS1 for activities that we generate or modify ourselves
|
2023-01-28 15:48:50 +00:00
|
|
|
|
2023-04-02 02:13:51 +00:00
|
|
|
# Protocol and subclasses set these in fetch if this Object is new or if its
|
|
|
|
# contents have changed from what was originally loaded from the datastore.
|
|
|
|
new = None
|
|
|
|
changed = None
|
|
|
|
|
2023-02-24 14:25:18 +00:00
|
|
|
@ComputedJsonProperty
|
2023-02-24 03:17:26 +00:00
|
|
|
def as1(self):
|
2023-03-21 16:10:14 +00:00
|
|
|
# TODO: switch back to assert?
|
2023-02-24 16:02:06 +00:00
|
|
|
# assert (self.as2 is not None) ^ (self.bsky is not None) ^ (self.mf2 is not None), \
|
|
|
|
# f'{self.as2} {self.bsky} {self.mf2}'
|
2023-03-08 21:10:41 +00:00
|
|
|
if bool(self.as2) + bool(self.bsky) + bool(self.mf2) > 1:
|
2023-04-03 14:53:15 +00:00
|
|
|
logger.warning(f'{self.key} has multiple! {bool(self.as2)} {bool(self.bsky)} {bool(self.mf2)}')
|
2023-02-24 16:02:06 +00:00
|
|
|
|
2023-03-21 16:10:14 +00:00
|
|
|
if self.our_as1 is not None:
|
2023-05-31 17:47:09 +00:00
|
|
|
return redirect_unwrap(self.our_as1)
|
2023-03-21 16:10:14 +00:00
|
|
|
elif self.as2 is not None:
|
2023-05-31 17:47:09 +00:00
|
|
|
return as2.to_as1(redirect_unwrap(self.as2))
|
2023-02-24 13:25:29 +00:00
|
|
|
elif self.bsky is not None:
|
|
|
|
return bluesky.to_as1(self.bsky)
|
|
|
|
elif self.mf2 is not None:
|
2023-04-05 01:02:41 +00:00
|
|
|
return microformats2.json_to_object(self.mf2,
|
|
|
|
rel_urls=self.mf2.get('rel-urls'))
|
2023-02-24 03:17:26 +00:00
|
|
|
|
2023-02-14 05:43:49 +00:00
|
|
|
@ndb.ComputedProperty
|
|
|
|
def type(self): # AS1 objectType, or verb if it's an activity
|
2023-02-25 03:59:12 +00:00
|
|
|
if self.as1:
|
|
|
|
return as1.object_type(self.as1)
|
2023-02-14 05:43:49 +00:00
|
|
|
|
|
|
|
def _object_ids(self): # id(s) of inner objects
|
2023-02-25 03:59:12 +00:00
|
|
|
if self.as1:
|
2023-05-31 17:47:09 +00:00
|
|
|
return redirect_unwrap(as1.get_ids(self.as1, 'object'))
|
2023-02-14 05:43:49 +00:00
|
|
|
object_ids = ndb.ComputedProperty(_object_ids, repeated=True)
|
|
|
|
|
2023-02-01 05:00:07 +00:00
|
|
|
deleted = ndb.BooleanProperty()
|
2023-01-28 15:48:50 +00:00
|
|
|
|
2023-02-01 21:19:41 +00:00
|
|
|
delivered = ndb.StructuredProperty(Target, repeated=True)
|
|
|
|
undelivered = ndb.StructuredProperty(Target, repeated=True)
|
|
|
|
failed = ndb.StructuredProperty(Target, repeated=True)
|
2018-11-20 16:22:26 +00:00
|
|
|
|
2023-02-03 04:03:55 +00:00
|
|
|
created = ndb.DateTimeProperty(auto_now_add=True)
|
|
|
|
updated = ndb.DateTimeProperty(auto_now=True)
|
2017-10-11 05:42:19 +00:00
|
|
|
|
2023-04-18 16:08:45 +00:00
|
|
|
# For certain types, automatically delete this Object after 90d using a
|
|
|
|
# TTL policy:
|
|
|
|
# https://cloud.google.com/datastore/docs/ttl#ttl_properties_and_indexes
|
|
|
|
# They recommend not indexing TTL properties:
|
|
|
|
# https://cloud.google.com/datastore/docs/ttl#ttl_properties_and_indexes
|
|
|
|
def _expire(self):
|
|
|
|
if self.type in OBJECT_EXPIRE_TYPES:
|
|
|
|
return (self.updated or util.now()) + OBJECT_EXPIRE_AGE
|
|
|
|
expire = ndb.ComputedProperty(_expire, indexed=False)
|
|
|
|
|
2023-03-28 04:51:18 +00:00
|
|
|
def _pre_put_hook(self):
|
2023-04-05 23:23:49 +00:00
|
|
|
assert '^^' not in self.key.id()
|
|
|
|
|
2023-03-28 04:51:18 +00:00
|
|
|
if self.as1 and self.as1.get('objectType') == 'activity':
|
|
|
|
if 'activity' not in self.labels:
|
|
|
|
self.labels.append('activity')
|
|
|
|
else:
|
|
|
|
if 'activity' in self.labels:
|
|
|
|
self.labels.remove('activity')
|
|
|
|
|
2023-02-14 22:30:00 +00:00
|
|
|
def _post_put_hook(self, future):
|
2023-03-29 20:13:32 +00:00
|
|
|
"""Update :meth:`Protocol.load` cache."""
|
2023-02-14 22:56:27 +00:00
|
|
|
# TODO: assert that as1 id is same as key id? in pre put hook?
|
2023-02-14 23:38:42 +00:00
|
|
|
logger.info(f'Wrote Object {self.key.id()} {self.type} {self.status or ""} {self.labels} for {len(self.domains)} users')
|
2023-03-28 04:51:18 +00:00
|
|
|
if '#' not in self.key.id():
|
2023-04-19 00:17:48 +00:00
|
|
|
import protocol # TODO: actually fix this circular import
|
2023-04-03 03:36:23 +00:00
|
|
|
protocol.objects_cache[self.key.id()] = self
|
2023-03-08 21:10:41 +00:00
|
|
|
|
2023-04-05 23:23:49 +00:00
|
|
|
@classmethod
|
|
|
|
def get_by_id(cls, id):
|
|
|
|
"""Override Model.get_by_id to un-escape ^^ to #.
|
|
|
|
|
|
|
|
https://github.com/snarfed/bridgy-fed/issues/469
|
|
|
|
|
|
|
|
See "meth:`proxy_url()` for the inverse.
|
|
|
|
"""
|
|
|
|
return super().get_by_id(id.replace('^^', '#'))
|
|
|
|
|
2023-03-08 21:10:41 +00:00
|
|
|
def clear(self):
|
|
|
|
"""Clears all data properties."""
|
|
|
|
for prop in 'as2', 'bsky', 'mf2':
|
|
|
|
val = getattr(self, prop, None)
|
|
|
|
if val:
|
2023-04-03 14:53:15 +00:00
|
|
|
logger.warning(f'Wiping out {prop}: {json_dumps(val, indent=2)}')
|
2023-03-08 21:10:41 +00:00
|
|
|
setattr(self, prop, None)
|
2023-02-14 22:30:00 +00:00
|
|
|
|
2021-07-09 05:50:33 +00:00
|
|
|
def proxy_url(self):
|
2023-04-05 23:23:49 +00:00
|
|
|
"""Returns the Bridgy Fed proxy URL to render this post as HTML.
|
|
|
|
|
|
|
|
Escapes # characters to ^^.
|
|
|
|
https://github.com/snarfed/bridgy-fed/issues/469
|
|
|
|
|
|
|
|
See "meth:`get_by_id()` for the inverse.
|
|
|
|
"""
|
|
|
|
assert '^^' not in self.key.id()
|
|
|
|
id = self.key.id().replace('#', '^^')
|
2023-05-30 19:15:36 +00:00
|
|
|
return common.host_url(f'convert/{self.source_protocol}/web/{id}')
|
2022-11-21 04:49:55 +00:00
|
|
|
|
2023-03-20 21:28:14 +00:00
|
|
|
def actor_link(self):
|
|
|
|
"""Returns a pretty actor link with their name and profile picture."""
|
2023-03-14 13:54:16 +00:00
|
|
|
attrs = {'class': 'h-card u-author'}
|
|
|
|
|
2023-05-30 19:15:36 +00:00
|
|
|
if (self.source_protocol in ('web', 'webmention', 'ui') and g.user and
|
2023-03-20 21:28:14 +00:00
|
|
|
g.user.key.id() in self.domains):
|
2023-02-07 04:30:08 +00:00
|
|
|
# outbound; show a nice link to the user
|
2023-03-20 21:28:14 +00:00
|
|
|
return g.user.user_page_link()
|
2022-11-26 06:21:50 +00:00
|
|
|
|
2023-02-24 03:17:26 +00:00
|
|
|
actor = (util.get_first(self.as1, 'actor')
|
|
|
|
or util.get_first(self.as1, 'author')
|
2023-02-03 16:10:09 +00:00
|
|
|
or {})
|
2022-11-26 06:21:50 +00:00
|
|
|
if isinstance(actor, str):
|
2023-03-20 21:28:14 +00:00
|
|
|
return common.pretty_link(actor, attrs=attrs)
|
2022-11-26 06:21:50 +00:00
|
|
|
|
2022-11-28 15:59:30 +00:00
|
|
|
url = util.get_first(actor, 'url') or ''
|
2023-02-07 16:24:24 +00:00
|
|
|
name = actor.get('displayName') or actor.get('username') or ''
|
2023-03-14 13:54:16 +00:00
|
|
|
image = util.get_url(actor, 'image')
|
2022-11-26 06:21:50 +00:00
|
|
|
if not image:
|
2023-03-20 21:28:14 +00:00
|
|
|
return common.pretty_link(url, text=name, attrs=attrs)
|
2022-11-26 06:21:50 +00:00
|
|
|
|
|
|
|
return f"""\
|
2023-03-14 13:54:16 +00:00
|
|
|
<a class="h-card u-author" href="{url}" title="{name}">
|
2022-11-26 06:21:50 +00:00
|
|
|
<img class="profile" src="{image}" />
|
|
|
|
{util.ellipsize(name, chars=40)}
|
|
|
|
</a>"""
|
|
|
|
|
2023-04-28 19:02:26 +00:00
|
|
|
class AtpNode(StringIdModel):
|
|
|
|
"""An AT Protocol (Bluesky) node.
|
|
|
|
|
|
|
|
May be a data record, an MST node, or a commit.
|
|
|
|
|
|
|
|
Key name is the DAG-CBOR base32 CID of the data.
|
|
|
|
|
|
|
|
Properties:
|
|
|
|
* data: JSON-decoded DAG-JSON value of this node
|
|
|
|
* obj: optional, Key of the corresponding :class:`Object`, only populated
|
|
|
|
for records
|
|
|
|
"""
|
|
|
|
data = JsonProperty(required=True)
|
|
|
|
obj = ndb.KeyProperty(Object)
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def create(data):
|
|
|
|
"""Writes a new AtpNode to the datastore.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
data: dict value
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
:class:`AtpNode`
|
|
|
|
"""
|
|
|
|
data = json.loads(dag_json.encode(data))
|
|
|
|
cid = dag_cbor_cid(data)
|
|
|
|
node = AtpNode(id=cid.encode('base32'), data=data)
|
|
|
|
node.put()
|
|
|
|
return node
|
|
|
|
|
2018-10-22 00:37:33 +00:00
|
|
|
|
|
|
|
class Follower(StringIdModel):
|
|
|
|
"""A follower of a Bridgy Fed user.
|
|
|
|
|
2022-11-11 23:44:35 +00:00
|
|
|
Key name is 'TO FROM', where each part is either a domain or an AP id, eg:
|
|
|
|
'snarfed.org https://mastodon.social/@swentel'.
|
|
|
|
|
|
|
|
Both parts are duplicated in the src and dest properties.
|
2018-10-22 00:37:33 +00:00
|
|
|
"""
|
2019-08-01 14:32:45 +00:00
|
|
|
STATUSES = ('active', 'inactive')
|
|
|
|
|
2022-11-11 23:44:35 +00:00
|
|
|
src = ndb.StringProperty()
|
|
|
|
dest = ndb.StringProperty()
|
2023-01-19 05:09:43 +00:00
|
|
|
# Most recent AP (AS2) JSON Follow activity. If inbound, must have a
|
|
|
|
# composite actor object with an inbox, publicInbox, or sharedInbox.
|
2023-02-24 13:25:29 +00:00
|
|
|
last_follow = JsonProperty()
|
2019-08-01 14:32:45 +00:00
|
|
|
status = ndb.StringProperty(choices=STATUSES, default='active')
|
2018-10-22 00:37:33 +00:00
|
|
|
|
|
|
|
created = ndb.DateTimeProperty(auto_now_add=True)
|
|
|
|
updated = ndb.DateTimeProperty(auto_now=True)
|
|
|
|
|
2023-02-14 23:38:42 +00:00
|
|
|
def _post_put_hook(self, future):
|
|
|
|
logger.info(f'Wrote Follower {self.key.id()} {self.status}')
|
|
|
|
|
2018-10-22 00:37:33 +00:00
|
|
|
@classmethod
|
2022-11-11 23:44:35 +00:00
|
|
|
def _id(cls, dest, src):
|
|
|
|
assert src
|
|
|
|
assert dest
|
2023-01-24 20:17:24 +00:00
|
|
|
return f'{dest} {src}'
|
2018-10-22 00:37:33 +00:00
|
|
|
|
|
|
|
@classmethod
|
2022-11-11 23:44:35 +00:00
|
|
|
def get_or_create(cls, dest, src, **kwargs):
|
2022-11-16 14:47:20 +00:00
|
|
|
follower = cls.get_or_insert(cls._id(dest, src), src=src, dest=dest, **kwargs)
|
|
|
|
follower.dest = dest
|
|
|
|
follower.src = src
|
2022-11-11 16:00:00 +00:00
|
|
|
for prop, val in kwargs.items():
|
|
|
|
setattr(follower, prop, val)
|
|
|
|
follower.put()
|
2022-11-16 14:47:20 +00:00
|
|
|
return follower
|
2023-01-13 19:40:52 +00:00
|
|
|
|
|
|
|
def to_as1(self):
|
|
|
|
"""Returns this follower as an AS1 actor dict, if possible."""
|
2023-01-19 14:49:39 +00:00
|
|
|
return as2.to_as1(self.to_as2())
|
|
|
|
|
|
|
|
def to_as2(self):
|
|
|
|
"""Returns this follower as an AS2 actor dict, if possible."""
|
2023-01-13 19:40:52 +00:00
|
|
|
if self.last_follow:
|
2023-02-24 13:25:29 +00:00
|
|
|
return self.last_follow.get('actor' if util.is_web(self.src) else 'object')
|
2023-03-08 21:10:41 +00:00
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def fetch_page(domain, collection):
|
|
|
|
"""Fetches a page of Follower entities.
|
|
|
|
|
|
|
|
Wraps :func:`fetch_page`. Paging uses the `before` and `after` query
|
|
|
|
parameters, if available in the request.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
domain: str, user to fetch entities for
|
|
|
|
collection, str, 'followers' or 'following'
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
(results, new_before, new_after) tuple with:
|
|
|
|
results: list of Follower entities
|
|
|
|
new_before, new_after: str query param values for `before` and `after`
|
|
|
|
to fetch the previous and next pages, respectively
|
|
|
|
"""
|
|
|
|
assert collection in ('followers', 'following'), collection
|
|
|
|
|
|
|
|
domain_prop = Follower.dest if collection == 'followers' else Follower.src
|
|
|
|
query = Follower.query(
|
|
|
|
Follower.status == 'active',
|
|
|
|
domain_prop == domain,
|
|
|
|
).order(-Follower.updated)
|
|
|
|
return fetch_page(query, Follower)
|
|
|
|
|
|
|
|
|
|
|
|
def fetch_page(query, model_class):
|
|
|
|
"""Fetches a page of results from a datastore query.
|
|
|
|
|
|
|
|
Uses the `before` and `after` query params (if provided; should be ISO8601
|
|
|
|
timestamps) and the queried model class's `updated` property to identify the
|
|
|
|
page to fetch.
|
|
|
|
|
|
|
|
Populates a `log_url_path` property on each result entity that points to a
|
|
|
|
its most recent logged request.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
query: :class:`ndb.Query`
|
|
|
|
model_class: ndb model class
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
(results, new_before, new_after) tuple with:
|
|
|
|
results: list of query result entities
|
|
|
|
new_before, new_after: str query param values for `before` and `after`
|
|
|
|
to fetch the previous and next pages, respectively
|
|
|
|
"""
|
|
|
|
# if there's a paging param ('before' or 'after'), update query with it
|
|
|
|
# TODO: unify this with Bridgy's user page
|
|
|
|
def get_paging_param(param):
|
|
|
|
val = request.values.get(param)
|
|
|
|
if val:
|
|
|
|
try:
|
|
|
|
dt = util.parse_iso8601(val.replace(' ', '+'))
|
|
|
|
except BaseException as e:
|
|
|
|
error(f"Couldn't parse {param}, {val!r} as ISO8601: {e}")
|
|
|
|
if dt.tzinfo:
|
|
|
|
dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
|
|
|
|
return dt
|
|
|
|
|
|
|
|
before = get_paging_param('before')
|
|
|
|
after = get_paging_param('after')
|
|
|
|
if before and after:
|
|
|
|
error("can't handle both before and after")
|
|
|
|
elif after:
|
|
|
|
query = query.filter(model_class.updated >= after).order(model_class.updated)
|
|
|
|
elif before:
|
|
|
|
query = query.filter(model_class.updated < before).order(-model_class.updated)
|
|
|
|
else:
|
|
|
|
query = query.order(-model_class.updated)
|
|
|
|
|
|
|
|
query_iter = query.iter()
|
|
|
|
results = sorted(itertools.islice(query_iter, 0, PAGE_SIZE),
|
|
|
|
key=lambda r: r.updated, reverse=True)
|
|
|
|
|
|
|
|
# calculate new paging param(s)
|
|
|
|
has_next = results and query_iter.probably_has_next()
|
|
|
|
new_after = (
|
|
|
|
before if before
|
|
|
|
else results[0].updated if has_next and after
|
|
|
|
else None)
|
|
|
|
if new_after:
|
|
|
|
new_after = new_after.isoformat()
|
|
|
|
|
|
|
|
new_before = (
|
|
|
|
after if after else
|
|
|
|
results[-1].updated if has_next
|
|
|
|
else None)
|
|
|
|
if new_before:
|
|
|
|
new_before = new_before.isoformat()
|
|
|
|
|
|
|
|
return results, new_before, new_after
|