mirror
/
bridgy-fed
kopia lustrzana https://github.com/snarfed/bridgy-fed
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

switch circular imports to runtime imports; split out flask_app.py from app.py 

runtime imports are just as bad, but...meh. eventually I'll untangle them for real. #486
pull/489/head
Ryan Barrett 2023-04-18 17:17:48 -07:00
rodzic ae76dd53e6
commit d2ab48b23e
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 6BE31FDF4776E9D4
25 zmienionych plików z 136 dodań i 136 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
activitypub.py
Wyświetl plik

@ -16,7 +16,7 @@ from oauth_dropins.webutil.util import fragmentless, json_dumps, json_loads
import requests
from werkzeug.exceptions import BadGateway
from app import app, cache
from flask_app import app, cache
import common
from common import (
CACHE_TIME,

67
app.py
Wyświetl plik

@ -1,64 +1,9 @@
"""Main Flask application."""
import json
import logging
from pathlib import Path
"""Bridgy Fed user-facing app invoked by gunicorn in app.yaml.
from flask import Flask, g
from flask_caching import Cache
import flask_gae_static
from lexrpc.server import Server
from lexrpc.flask_server import init_flask
from oauth_dropins.webutil import (
appengine_info,
appengine_config,
flask_util,
util,
)
import common
logger = logging.getLogger(__name__)
logging.getLogger('lexrpc').setLevel(logging.INFO)
logging.getLogger('negotiator').setLevel(logging.WARNING)
app_dir = Path(__file__).parent
app = Flask(__name__, static_folder=None)
app.template_folder = './templates'
app.json.compact = False
app.config.from_pyfile(app_dir / 'config.py')
app.url_map.converters['regex'] = flask_util.RegexConverter
app.after_request(flask_util.default_modern_headers)
app.register_error_handler(Exception, flask_util.handle_exception)
if appengine_info.LOCAL:
flask_gae_static.init_app(app)
@app.before_request
def init_globals():
g.user = None
# don't redirect API requests with blank path elements
app.url_map.redirect_defaults = True
app.wsgi_app = flask_util.ndb_context_middleware(
app.wsgi_app, client=appengine_config.ndb_client)
cache = Cache(app)
util.set_user_agent('Bridgy Fed (https://fed.brid.gy/)')
# XRPC server
lexicons = []
for filename in (app_dir / 'lexicons/app/bsky').glob('**/*.json'):
with open(filename) as f:
lexicons.append(json.load(f))
xrpc_server = Server(lexicons, validate=False)
init_flask(xrpc_server, app)
Import all modules that define views in the app so that their URL routes get
registered.
"""
from flask_app import app
# import all modules to register their Flask handlers
import activitypub
import follow, pages
import redirect
import render, superfeedr, webfinger, webmention, xrpc_actor, xrpc_feed, xrpc_graph
import activitypub, follow, pages, redirect, render, superfeedr, webfinger, webmention, xrpc_actor, xrpc_feed, xrpc_graph

58
flask_app.py 100644
Wyświetl plik

@ -0,0 +1,58 @@
"""Main Flask application."""
import json
import logging
from pathlib import Path
from flask import Flask, g
from flask_caching import Cache
import flask_gae_static
from lexrpc.server import Server
from lexrpc.flask_server import init_flask
from oauth_dropins.webutil import (
appengine_info,
appengine_config,
flask_util,
util,
)
import common
logger = logging.getLogger(__name__)
logging.getLogger('lexrpc').setLevel(logging.INFO)
logging.getLogger('negotiator').setLevel(logging.WARNING)
app_dir = Path(__file__).parent
app = Flask(__name__, static_folder=None)
app.template_folder = './templates'
app.json.compact = False
app.config.from_pyfile(app_dir / 'config.py')
app.url_map.converters['regex'] = flask_util.RegexConverter
app.after_request(flask_util.default_modern_headers)
app.register_error_handler(Exception, flask_util.handle_exception)
if appengine_info.LOCAL:
flask_gae_static.init_app(app)
@app.before_request
def init_globals():
g.user = None
# don't redirect API requests with blank path elements
app.url_map.redirect_defaults = True
app.wsgi_app = flask_util.ndb_context_middleware(
app.wsgi_app, client=appengine_config.ndb_client)
cache = Cache(app)
util.set_user_agent('Bridgy Fed (https://fed.brid.gy/)')
# XRPC server
lexicons = []
for filename in (app_dir / 'lexicons/app/bsky').glob('**/*.json'):
with open(filename) as f:
lexicons.append(json.load(f))
xrpc_server = Server(lexicons, validate=False)
init_flask(xrpc_server, app)

13
follow.py
Wyświetl plik

@ -16,9 +16,8 @@ from oauth_dropins.webutil import util
from oauth_dropins.webutil.testutil import NOW
from oauth_dropins.webutil.util import json_dumps, json_loads
# import module instead of ActivityPub class to avoid circular import
import activitypub
from app import app
from activitypub import ActivityPub
from flask_app import app
import common
import models
@ -159,7 +158,7 @@ class FollowCallback(indieauth.Callback):
return redirect(f'/user/{domain}/following')
# TODO: make this generic across protocols
followee = activitypub.ActivityPub.load(as2_url).as2
followee = ActivityPub.load(as2_url).as2
id = followee.get('id')
inbox = followee.get('inbox')
if not id or not inbox:
@ -178,7 +177,7 @@ class FollowCallback(indieauth.Callback):
}
obj = models.Object(id=follow_id, domains=[domain], labels=['user'],
source_protocol='ui', status='complete', as2=follow_as2)
activitypub.ActivityPub.send(obj, inbox)
ActivityPub.send(obj, inbox)
models.Follower.get_or_create(dest=id, src=domain, status='active',
last_follow=follow_as2)
@ -237,7 +236,7 @@ class UnfollowCallback(indieauth.Callback):
if isinstance(followee, str):
# fetch as AS2 to get full followee with inbox
followee_id = followee
followee = activitypub.ActivityPub.load(followee_id).as2
followee = ActivityPub.load(followee_id).as2
inbox = followee.get('inbox')
if not inbox:
@ -256,7 +255,7 @@ class UnfollowCallback(indieauth.Callback):
obj = models.Object(id=unfollow_id, domains=[domain], labels=['user'],
source_protocol='ui', status='complete', as2=unfollow_as2)
activitypub.ActivityPub.send(obj, inbox)
ActivityPub.send(obj, inbox)
follower.status = 'inactive'
follower.put()

6
models.py
Wyświetl plik

@ -20,11 +20,7 @@ from oauth_dropins.webutil.models import ComputedJsonProperty, JsonProperty, Str
from oauth_dropins.webutil import util
from oauth_dropins.webutil.util import json_dumps, json_loads
# import top level modules to avoid circular imports
import activitypub
import common
import protocol
import webmention
# https://github.com/snarfed/bridgy-fed/issues/314
WWW_DOMAINS = frozenset((
@ -260,6 +256,7 @@ class User(StringIdModel):
# check home page
try:
import activitypub, webmention # TODO: actually fix these circular imports
obj = webmention.Webmention.load(self.homepage, gateway=True)
self.actor_as2 = activitypub.postprocess_as2(as2.from_as1(obj.as1))
self.has_hcard = True
@ -380,6 +377,7 @@ class Object(StringIdModel):
# TODO: assert that as1 id is same as key id? in pre put hook?
logger.info(f'Wrote Object {self.key.id()} {self.type} {self.status or ""} {self.labels} for {len(self.domains)} users')
if '#' not in self.key.id():
import protocol # TODO: actually fix this circular import
protocol.objects_cache[self.key.id()] = self
@classmethod

2
pages.py
Wyświetl plik

@ -14,7 +14,7 @@ from oauth_dropins.webutil import flask_util, logs, util
from oauth_dropins.webutil.flask_util import error, flash, redirect
from oauth_dropins.webutil.util import json_dumps, json_loads
from app import app, cache
from flask_app import app, cache
import common
from common import DOMAIN_RE
from models import fetch_page, Follower, Object, PAGE_SIZE, User

39
protocol.py
Wyświetl plik

@ -10,8 +10,7 @@ from granary import as1, as2
import common
from common import error
# import module instead of individual classes to avoid circular import
import models
from models import Follower, Object, Target, User
from oauth_dropins.webutil import util, webmention
from oauth_dropins.webutil.util import json_dumps, json_loads
@ -119,12 +118,12 @@ class Protocol:
with seen_ids_lock:
already_seen = id in seen_ids
seen_ids[id] = True
if already_seen or models.Object.get_by_id(id):
if already_seen or Object.get_by_id(id):
msg = f'Already handled this activity {id}'
logger.info(msg)
return msg, 200
obj = models.Object.get_or_insert(id)
obj = Object.get_or_insert(id)
obj.clear()
obj.populate(source_protocol=cls.LABEL, **props)
obj.put()
@ -138,8 +137,8 @@ class Protocol:
inner_obj = as1.get_object(obj.as1)
inner_obj_id = inner_obj.get('id')
if obj.type in ('post', 'create', 'update') and inner_obj.keys() > set(['id']):
to_update = (models.Object.get_by_id(inner_obj_id)
or models.Object(id=inner_obj_id))
to_update = (Object.get_by_id(inner_obj_id)
or Object(id=inner_obj_id))
to_update.populate(as2=obj.as2['object'], source_protocol=cls.LABEL)
to_update.put()
@ -156,8 +155,8 @@ class Protocol:
# deactivate Follower
followee_domain = util.domain_from_link(inner_obj_id, minimize=False)
follower = models.Follower.get_by_id(
models.Follower._id(dest=followee_domain, src=actor_id))
follower = Follower.get_by_id(
Follower._id(dest=followee_domain, src=actor_id))
if follower:
logger.info(f'Marking {follower} inactive')
follower.status = 'inactive'
@ -183,7 +182,7 @@ class Protocol:
if not inner_obj_id:
error("Couldn't find id of object to delete")
to_delete = models.Object.get_by_id(inner_obj_id)
to_delete = Object.get_by_id(inner_obj_id)
if to_delete:
logger.info(f'Marking Object {inner_obj_id} deleted')
to_delete.deleted = True
@ -192,9 +191,9 @@ class Protocol:
# assume this is an actor
# https://github.com/snarfed/bridgy-fed/issues/63
logger.info(f'Deactivating Followers with src or dest = {inner_obj_id}')
followers = models.Follower.query(OR(models.Follower.src == inner_obj_id,
models.Follower.dest == inner_obj_id)
).fetch()
followers = Follower.query(OR(Follower.src == inner_obj_id,
Follower.dest == inner_obj_id)
).fetch()
for f in followers:
f.status = 'inactive'
obj.status = 'complete'
@ -222,9 +221,9 @@ class Protocol:
if (actor and actor_id and
(obj.type == 'share' or obj.type in ('create', 'post') and not is_reply)):
logger.info(f'Delivering to followers of {actor_id}')
for f in models.Follower.query(models.Follower.dest == actor_id,
models.Follower.status == 'active',
projection=[models.Follower.src]):
for f in Follower.query(Follower.dest == actor_id,
Follower.status == 'active',
projection=[Follower.src]):
if f.src not in obj.domains:
obj.domains.append(f.src)
if obj.domains and 'feed' not in obj.labels:
@ -254,7 +253,7 @@ class Protocol:
error(f'Follow actor requires id and inbox. Got: {follower}')
# store Follower
follower_obj = models.Follower.get_or_create(
follower_obj = Follower.get_or_create(
dest=g.user.key.id(), src=follower_id, last_follow=obj.as2)
follower_obj.status = 'active'
follower_obj.put()
@ -272,7 +271,7 @@ class Protocol:
'object': followee_actor_url,
}
}
return cls.send(models.Object(as2=accept), inbox)
return cls.send(Object(as2=accept), inbox)
@classmethod
def deliver(cls, obj):
@ -314,7 +313,7 @@ class Protocol:
# send webmentions and update Object
errors = [] # stores (code, body) tuples
targets = [models.Target(uri=uri, protocol='webmention') for uri in targets]
targets = [Target(uri=uri, protocol='webmention') for uri in targets]
obj.populate(
undelivered=targets,
@ -394,7 +393,7 @@ class Protocol:
logger.info(f'Loading Object {id}')
orig_as1 = None
obj = models.Object.get_by_id(id)
obj = Object.get_by_id(id)
if obj and (obj.as1 or obj.deleted):
logger.info(' got from datastore')
obj.new = False
@ -411,7 +410,7 @@ class Protocol:
obj.clear()
else:
logger.info(f' not in datastore')
obj = models.Object(id=id)
obj = Object(id=id)
obj.new = True
obj.changed = False

3
redirect.py
Wyświetl plik

@ -21,9 +21,8 @@ from oauth_dropins.webutil import flask_util, util
from oauth_dropins.webutil.flask_util import error
from oauth_dropins.webutil.util import json_dumps, json_loads
# import module instead of individual functions to avoid circular import
import activitypub
from app import app, cache
from flask_app import app, cache
from common import CACHE_TIME, CONTENT_TYPE_HTML
from models import Object, User

2
render.py
Wyświetl plik

@ -10,7 +10,7 @@ from oauth_dropins.webutil.flask_util import error
from oauth_dropins.webutil import util
import activitypub
from app import app, cache
from flask_app import app, cache
import common
from models import Object

2
superfeedr.py
Wyświetl plik

@ -8,7 +8,7 @@ import logging
from flask import request
from app import app
from flask_app import app
logger = logging.getLogger(__name__)

2
tests/test_activitypub.py
Wyświetl plik

@ -21,7 +21,7 @@ from werkzeug.exceptions import BadGateway
import activitypub
from activitypub import ActivityPub
from app import app
from flask_app import app
import common
import models
from models import Follower, Object, User

2
tests/test_common.py
Wyświetl plik

@ -7,7 +7,7 @@ from oauth_dropins.webutil import appengine_config, util
from oauth_dropins.webutil.testutil import requests_response
import requests
from app import app
from flask_app import app
import common
from models import Object, User
import protocol

2
tests/test_follow.py
Wyświetl plik

@ -10,7 +10,7 @@ from oauth_dropins.webutil import util
from oauth_dropins.webutil.testutil import requests_response
from oauth_dropins.webutil.util import json_dumps, json_loads
from app import app
from flask_app import app
import common
from common import redirect_unwrap
from models import Follower, Object, User

2
tests/test_pages.py
Wyświetl plik

@ -15,7 +15,7 @@ from granary.tests.test_as1 import (
from oauth_dropins.webutil import util
from oauth_dropins.webutil.testutil import requests_response
from app import app
from flask_app import app
import common
from models import Object, Follower, User
from . import testutil

2
tests/test_protocol.py
Wyświetl plik

@ -6,7 +6,7 @@ from oauth_dropins.webutil.testutil import requests_response
import requests
from protocol import Protocol
from app import app
from flask_app import app
from models import Follower, Object, User
from .test_activitypub import ACTOR, REPLY

2
tests/test_redirect.py
Wyświetl plik

@ -4,7 +4,7 @@ import copy
from granary import as2
from app import app, cache
from flask_app import app, cache
from common import redirect_unwrap
from models import Object, User
from .test_webmention import REPOST_AS2

2
tests/test_render.py
Wyświetl plik

@ -4,7 +4,7 @@ import copy
from granary import as2
from granary.tests.test_as1 import ACTOR, COMMENT, DELETE_OF_ID, UPDATE
from app import app
from flask_app import app
import common
from models import Object
import render

2
tests/test_webmention.py
Wyświetl plik

@ -17,7 +17,7 @@ import requests
from werkzeug.exceptions import BadGateway, BadRequest
import activitypub
from app import app
from flask_app import app
from common import (
CONTENT_TYPE_HTML,
redirect_unwrap,

2
tests/test_xrpc_feed.py
Wyświetl plik

@ -17,7 +17,7 @@ from oauth_dropins.webutil.testutil import requests_response
import requests
from werkzeug.exceptions import BadGateway
from app import app
from flask_app import app
import common
from models import Object, User
from . import testutil

5
tests/testutil.py
Wyświetl plik

@ -18,11 +18,14 @@ from oauth_dropins.webutil.appengine_config import ndb_client
from oauth_dropins.webutil.testutil import requests_response
import requests
from app import app, cache
# load all Flask handlers
import app
from flask_app import app, cache
import activitypub, common
from models import Object, PROTOCOLS, Target, User
import protocol
logger = logging.getLogger(__name__)
# used in TestCase.make_user() to reuse RSA keys across Users

2
webfinger.py
Wyświetl plik

@ -14,7 +14,7 @@ from oauth_dropins.webutil import flask_util, util
from oauth_dropins.webutil.flask_util import error
from oauth_dropins.webutil.util import json_dumps, json_loads
from app import app, cache
from flask_app import app, cache
import common
from models import User

47
webmention.py
Wyświetl plik

@ -18,10 +18,10 @@ from oauth_dropins.webutil import webmention
from requests import HTTPError, RequestException, URLRequired
from werkzeug.exceptions import BadGateway, BadRequest, HTTPException
# import module instead of individual classes/functions to avoid circular import
import activitypub
from app import app
from activitypub import ActivityPub
from flask_app import app
import common
from models import Follower, Object, Target, User
import models
from protocol import Protocol
@ -132,7 +132,7 @@ def webmention_external():
error(f'Bad URL {source}')
domain = util.domain_from_link(source, minimize=False)
g.user = models.User.get_by_id(domain)
g.user = User.get_by_id(domain)
if not g.user:
error(f'No user found for domain {domain}')
@ -180,7 +180,7 @@ def webmention_task():
domain = util.domain_from_link(source, minimize=False)
logger.info(f'webmention from {domain}')
g.user = models.User.get_by_id(domain)
g.user = User.get_by_id(domain)
if not g.user:
error(f'No user found for domain {domain}', status=304)
@ -195,12 +195,12 @@ def webmention_task():
create_id = f'{source}#bridgy-fed-create'
logger.info(f'Interpreting as Delete. Looking for {create_id}')
create = models.Object.get_by_id(create_id)
create = Object.get_by_id(create_id)
if not create or create.status != 'complete':
error(f"Bridgy Fed hasn't successfully published {source}", status=304)
id = f'{source}#bridgy-fed-delete'
obj = models.Object(id=id, our_as1={
obj = Object(id=id, our_as1={
'id': id,
'objectType': 'activity',
'verb': 'delete',
@ -227,7 +227,7 @@ def webmention_task():
'updated': util.now().isoformat(),
}
id = common.host_url(f'{obj.key.id()}#update-{util.now().isoformat()}')
obj = models.Object(id=id, our_as1={
obj = Object(id=id, our_as1={
'objectType': 'activity',
'verb': 'update',
'id': id,
@ -274,9 +274,8 @@ def webmention_task():
**obj.as1,
},
}
obj = models.Object(
id=id, mf2=obj.mf2, our_as1=update_as1, labels=['user'],
domains=[g.user.key.id()], source_protocol='webmention')
obj = Object(id=id, mf2=obj.mf2, our_as1=update_as1, labels=['user'],
domains=[g.user.key.id()], source_protocol='webmention')
elif obj.new:
logger.info(f'New Object {obj.key.id()}')
@ -290,9 +289,9 @@ def webmention_task():
'actor': g.user.actor_id(),
'object': obj.as1,
}
obj = models.Object(id=id, mf2=obj.mf2, our_as1=create_as1,
domains=[g.user.key.id()], labels=['user'],
source_protocol='webmention')
obj = Object(id=id, mf2=obj.mf2, our_as1=create_as1,
domains=[g.user.key.id()], labels=['user'],
source_protocol='webmention')
else:
msg = f'{obj.key.id()} is unchanged, nothing to do'
@ -307,7 +306,7 @@ def webmention_task():
labels=['user'],
delivered=[],
failed=[],
undelivered=[models.Target(uri=uri, protocol='activitypub')
undelivered=[Target(uri=uri, protocol='activitypub')
for uri in inboxes_to_targets.keys()],
)
@ -324,16 +323,16 @@ def webmention_task():
# prefer AS2 id or url, if available
# https://github.com/snarfed/bridgy-fed/issues/307
dest = target_as2 or as1.get_object(obj.as1)
models.Follower.get_or_create(dest=dest.get('id') or dest.get('url'),
src=g.user.key.id(),
last_follow=as2.from_as1(obj.as1))
Follower.get_or_create(dest=dest.get('id') or dest.get('url'),
src=g.user.key.id(),
last_follow=as2.from_as1(obj.as1))
# this is reused later in ActivityPub.send()
# TODO: find a better way
obj.target_as2 = target_as2
try:
last = activitypub.ActivityPub.send(obj, inbox, log_data=log_data)
last = ActivityPub.send(obj, inbox, log_data=log_data)
obj.delivered.append(target)
last_success = last
except BaseException as e:
@ -388,9 +387,9 @@ def _activitypub_targets(obj):
logger.info('Delivering to followers')
inboxes = set()
domain = g.user.key.id()
for follower in models.Follower.query().filter(
models.Follower.key > Key('Follower', domain + ' '),
models.Follower.key < Key('Follower', domain + CHAR_AFTER_SPACE)):
for follower in Follower.query().filter(
Follower.key > Key('Follower', domain + ' '),
Follower.key < Key('Follower', domain + CHAR_AFTER_SPACE)):
if follower.status != 'inactive' and follower.last_follow:
actor = follower.last_follow.get('actor')
if actor and isinstance(actor, dict):
@ -409,7 +408,7 @@ def _activitypub_targets(obj):
# fetch target page as AS2 object
try:
# TODO: make this generic across protocols
target_stored = activitypub.ActivityPub.load(target)
target_stored = ActivityPub.load(target)
target_obj = target_stored.as2 or as2.from_as1(target_stored.as1)
except (HTTPError, BadGateway) as e:
resp = getattr(e, 'requests_response', None)
@ -435,7 +434,7 @@ def _activitypub_targets(obj):
if not inbox_url:
# fetch actor as AS object
# TODO: make this generic across protocols
actor_obj = activitypub.ActivityPub.load(actor)
actor_obj = ActivityPub.load(actor)
actor = actor_obj.as2 or as2.from_as1(actor_obj.as1)
inbox_url = actor.get('inbox')

2
xrpc_actor.py
Wyświetl plik

@ -8,7 +8,7 @@ from granary import microformats2, bluesky
import mf2util
from oauth_dropins.webutil import util
from app import xrpc_server
from flask_app import xrpc_server
from models import User
logger = logging.getLogger(__name__)

2
xrpc_feed.py
Wyświetl plik

@ -8,7 +8,7 @@ from granary import bluesky, microformats2
import mf2util
from oauth_dropins.webutil import util
from app import xrpc_server
from flask_app import xrpc_server
from models import Object, PAGE_SIZE, User
logger = logging.getLogger(__name__)

2
xrpc_graph.py
Wyświetl plik

@ -5,7 +5,7 @@ import re
from granary import bluesky
from oauth_dropins.webutil import util
from app import xrpc_server
from flask_app import xrpc_server
import common
from models import Follower, User