2024-06-13 00:04:59 +00:00
|
|
|
"""Single-instance hub for ATProto subscription (firehose) server and client."""
|
2023-09-05 19:06:07 +00:00
|
|
|
import logging
|
|
|
|
import os
|
|
|
|
from pathlib import Path
|
2024-05-09 20:30:42 +00:00
|
|
|
import threading
|
|
|
|
from threading import Thread, Timer
|
2023-09-05 19:06:07 +00:00
|
|
|
|
|
|
|
import arroba.server
|
|
|
|
from arroba import xrpc_sync
|
2024-05-14 05:02:09 +00:00
|
|
|
from flask import Flask
|
2023-09-29 18:23:50 +00:00
|
|
|
import lexrpc.client
|
2023-09-05 19:06:07 +00:00
|
|
|
import lexrpc.flask_server
|
2024-05-09 20:30:42 +00:00
|
|
|
from oauth_dropins.webutil.appengine_info import DEBUG, LOCAL_SERVER
|
2024-06-13 00:04:59 +00:00
|
|
|
from oauth_dropins.webutil import appengine_config, flask_util
|
2023-09-05 19:06:07 +00:00
|
|
|
|
2024-05-09 20:30:42 +00:00
|
|
|
# all protocols
|
2024-05-14 05:02:09 +00:00
|
|
|
import activitypub, atproto, web
|
2024-06-13 00:04:59 +00:00
|
|
|
import atproto_firehose
|
2024-06-05 02:10:01 +00:00
|
|
|
from common import global_cache, global_cache_timeout_policy, USER_AGENT
|
2023-09-14 16:42:11 +00:00
|
|
|
import models
|
2023-09-05 19:06:07 +00:00
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
2023-09-14 16:42:11 +00:00
|
|
|
models.reset_protocol_properties()
|
2023-09-05 19:06:07 +00:00
|
|
|
|
|
|
|
# Flask app
|
|
|
|
app = Flask(__name__)
|
|
|
|
app.json.compact = False
|
|
|
|
app_dir = Path(__file__).parent
|
|
|
|
app.config.from_pyfile(app_dir / 'config.py')
|
|
|
|
|
|
|
|
app.wsgi_app = flask_util.ndb_context_middleware(
|
|
|
|
app.wsgi_app, client=appengine_config.ndb_client,
|
2024-06-05 02:10:01 +00:00
|
|
|
global_cache=global_cache,
|
|
|
|
global_cache_timeout_policy=global_cache_timeout_policy,
|
|
|
|
# disable context-local cache due to this bug:
|
|
|
|
# https://github.com/googleapis/python-ndb/issues/888
|
2024-06-01 14:07:00 +00:00
|
|
|
cache_policy=lambda key: False)
|
2023-09-05 19:06:07 +00:00
|
|
|
|
|
|
|
|
|
|
|
@app.get('/liveness_check')
|
|
|
|
@app.get('/readiness_check')
|
|
|
|
def health_check():
|
2023-09-09 21:01:29 +00:00
|
|
|
"""App Engine Flex health checks.
|
2023-09-05 19:06:07 +00:00
|
|
|
|
2023-09-09 21:01:29 +00:00
|
|
|
https://cloud.google.com/appengine/docs/flexible/reference/app-yaml?tab=python#updated_health_checks
|
|
|
|
"""
|
|
|
|
return 'OK'
|
2023-09-05 19:06:07 +00:00
|
|
|
|
|
|
|
|
2024-05-21 20:14:16 +00:00
|
|
|
# ATProto XRPC server
|
2023-09-05 19:06:07 +00:00
|
|
|
lexrpc.flask_server.init_flask(arroba.server.server, app)
|
|
|
|
|
2023-11-14 20:30:14 +00:00
|
|
|
|
2023-09-29 20:49:17 +00:00
|
|
|
@app.post('/queue/atproto-commit')
|
2023-10-17 18:33:56 +00:00
|
|
|
@flask_util.cloud_tasks_only
|
2023-09-09 21:01:29 +00:00
|
|
|
def atproto_commit():
|
2023-09-10 23:44:05 +00:00
|
|
|
"""Handler for atproto-commit tasks.
|
2023-09-09 21:01:29 +00:00
|
|
|
|
2023-09-10 23:44:05 +00:00
|
|
|
Triggers `subscribeRepos` to check for new commits.
|
|
|
|
"""
|
2024-05-22 23:47:09 +00:00
|
|
|
xrpc_sync.send_events()
|
2023-09-09 21:01:29 +00:00
|
|
|
return 'OK'
|
2023-09-29 18:23:50 +00:00
|
|
|
|
|
|
|
|
2024-06-13 00:04:59 +00:00
|
|
|
# start firehose consumer threads
|
|
|
|
if LOCAL_SERVER or not DEBUG:
|
|
|
|
threads = [t.name for t in threading.enumerate()]
|
|
|
|
assert 'atproto_firehose.subscriber' not in threads
|
|
|
|
assert 'atproto_firehose.handler' not in threads
|
|
|
|
|
|
|
|
Thread(target=atproto_firehose.subscriber, name='atproto_firehose.subscriber').start()
|
|
|
|
Thread(target=atproto_firehose.handler, name='atproto_firehose.handler').start()
|
|
|
|
|
|
|
|
|
2024-07-12 14:50:00 +00:00
|
|
|
# send requestCrawl to relay every 5m.
|
|
|
|
# delay 15s at startup because we're not up and serving XRPCs at this point yet.
|
|
|
|
# not sure why not.
|
2023-09-29 18:23:50 +00:00
|
|
|
if 'GAE_INSTANCE' in os.environ: # prod
|
|
|
|
def request_crawl():
|
|
|
|
bgs = lexrpc.client.Client(f'https://{os.environ["BGS_HOST"]}',
|
|
|
|
headers={'User-Agent': USER_AGENT})
|
|
|
|
resp = bgs.com.atproto.sync.requestCrawl({'hostname': os.environ['PDS_HOST']})
|
|
|
|
logger.info(resp)
|
2024-07-12 14:50:00 +00:00
|
|
|
Timer(5 * 60, request_crawl).start()
|
2023-09-29 18:23:50 +00:00
|
|
|
|
|
|
|
Timer(15, request_crawl).start()
|
2024-05-09 13:43:51 +00:00
|
|
|
logger.info('Will send relay requestCrawl in 15s')
|