From 0277675ceec2e70996bbf388f3446bc035c7ba39 Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Sat, 24 Dec 2022 08:34:24 -0800 Subject: [PATCH] initial code skeleton and app scaffolding, largely copied from Bridgy Fed --- .gcloudignore | 42 +++ .gitignore | 10 + actor.py | 8 + app.py | 36 +++ app.yaml | 50 ++++ common.py | 23 ++ config.py | 21 ++ config.yml | 58 ++++ dependabot.yml | 8 + feed.py | 8 + graph.py | 7 + models.py | 11 + pages.py | 146 ++++++++++ requirements.txt | 84 ++++++ static/robots.txt | 4 + templates/base.html | 68 +++++ templates/docs.html | 419 +++++++++++++++++++++++++++ templates/enter_web_site.html | 14 + templates/feed.html | 20 ++ templates/index.html | 48 +++ templates/paging.html | 13 + templates/stats.html | 16 + templates/user.html | 68 +++++ templates/user_addresses.html | 16 + templates/user_not_found.html | 14 + tests/__init__.py | 2 + tests/test_actor.py | 8 + tests/test_feed.py | 1 + tests/test_graph.py | 1 + tests/testutil.py | 33 +++ workflows/auto-merge-dependabot.yaml | 37 +++ workflows/codeql-analysis.yml | 64 ++++ workflows/dependency-review.yaml | 17 ++ 33 files changed, 1375 insertions(+) create mode 100644 .gcloudignore create mode 100644 .gitignore create mode 100644 actor.py create mode 100644 app.py create mode 100644 app.yaml create mode 100644 common.py create mode 100644 config.py create mode 100644 config.yml create mode 100644 dependabot.yml create mode 100644 feed.py create mode 100644 graph.py create mode 100644 models.py create mode 100644 pages.py create mode 100644 requirements.txt create mode 100644 static/robots.txt create mode 100644 templates/base.html create mode 100644 templates/docs.html create mode 100644 templates/enter_web_site.html create mode 100644 templates/feed.html create mode 100644 templates/index.html create mode 100644 templates/paging.html create mode 100644 templates/stats.html create mode 100644 templates/user.html create mode 100644 templates/user_addresses.html create mode 100644 templates/user_not_found.html create mode 100644 tests/__init__.py create mode 100644 tests/test_actor.py create mode 100644 tests/test_feed.py create mode 100644 tests/test_graph.py create mode 100644 tests/testutil.py create mode 100644 workflows/auto-merge-dependabot.yaml create mode 100644 workflows/codeql-analysis.yml create mode 100644 workflows/dependency-review.yaml diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 0000000..cb21a9f --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,42 @@ +# -*- conf -*- +# Duplicated in other App Engine project repos: Bridgy, Bridgy Fed, granary, +# oauth-dropins, etc. If you make a change here, change them too! +# +# https://cloud.google.com/appengine/docs/standard/python/config/appref#Python_app_yaml_Includes + +*.bak +*.c +*.cc +*.cpp +*.h +*.o +*.pyc +*.pyo +*.so +.git* + +__pycache__/ +.coverage/ +.git*/ +coverage/ +debian/ +doc/ +docs/ +example/ +examples/ +l/ +l3/ +local/ +local3/ +local3.7/ +pydoc/ +pydocs/ +python3/ +RCS/ +ref/ +sample/ +samples/ +TAGS +TAGS/ +test/ +tests/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c32464e --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.coverage +/.well-known/acme-challenge/ +datastore.dat* +/docs/_build/ +flask_secret_key +/l +/local* +private_notes +service_account_creds.json +TAGS diff --git a/actor.py b/actor.py new file mode 100644 index 0000000..db042aa --- /dev/null +++ b/actor.py @@ -0,0 +1,8 @@ +"""app.bsky.actor.* XRPC methods.""" + +# lexicons/app/bsky/actor/createScene.json +# lexicons/app/bsky/actor/getProfile.json +# lexicons/app/bsky/actor/getSuggestions.json +# lexicons/app/bsky/actor/search.json +# lexicons/app/bsky/actor/searchTypeahead.json +# lexicons/app/bsky/actor/updateProfile.json diff --git a/app.py b/app.py new file mode 100644 index 0000000..bd36303 --- /dev/null +++ b/app.py @@ -0,0 +1,36 @@ +"""Main Flask application.""" +from pathlib import Path + +from flask import Flask +from flask_caching import Cache +import flask_gae_static +from oauth_dropins.webutil import ( + appengine_info, + appengine_config, + flask_util, + util, +) + + +app = Flask(__name__, static_folder=None) +app.template_folder = './templates' +app.json.compact = False +app.config.from_pyfile(Path(__file__).parent / 'config.py') +app.url_map.converters['regex'] = flask_util.RegexConverter +app.after_request(flask_util.default_modern_headers) +app.register_error_handler(Exception, flask_util.handle_exception) +if appengine_info.LOCAL: + flask_gae_static.init_app(app) + +# don't redirect API requests with blank path elements +app.url_map.redirect_defaults = True + +app.wsgi_app = flask_util.ndb_context_middleware( + app.wsgi_app, client=appengine_config.ndb_client) + +cache = Cache(app) + +util.set_user_agent('Bridgy AT (https://at.brid.gy/)') + + +import pages diff --git a/app.yaml b/app.yaml new file mode 100644 index 0000000..23bd20a --- /dev/null +++ b/app.yaml @@ -0,0 +1,50 @@ +# https://cloud.google.com/appengine/docs/standard/python/config/appref + +# application: bridgy-at + +runtime: python39 + +# default_expiration: 1h + +# https://cloud.google.com/appengine/docs/standard/python3/runtime#entrypoint_best_practices +# https://docs.gunicorn.org/en/latest/settings.html#timeout +entrypoint: gunicorn --workers 1 --threads 10 --timeout 60 -b :$PORT app:app + +# background: https://github.com/snarfed/bridgy/issues/578 +# https://github.com/snarfed/bridgy/issues/1051 +automatic_scaling: + max_idle_instances: 1 + target_cpu_utilization: .9 + min_pending_latency: 3000ms + max_concurrent_requests: 30 + +inbound_services: +- warmup + +handlers: + +# static +- url: /static + static_dir: static + secure: always + +- url: /oauth_dropins_static + static_dir: oauth_dropins_static + +- url: /fonts + static_dir: oauth_dropins_fonts + +- url: /favicon.ico + static_files: static/favicon.ico + upload: static/favicon.ico + secure: always + +- url: /robots.txt + static_files: static/robots.txt + upload: static/robots.txt + secure: always + +# dynamic +- url: .* + script: auto + secure: always diff --git a/common.py b/common.py new file mode 100644 index 0000000..75e0257 --- /dev/null +++ b/common.py @@ -0,0 +1,23 @@ +"""Misc common utilities.""" +import datetime + +DOMAIN_RE = r'([^/:]+\.[^/:]+)' +TLD_BLOCKLIST = ('7z', 'asp', 'aspx', 'gif', 'html', 'ico', 'jpg', 'jpeg', 'js', + 'json', 'php', 'png', 'rar', 'txt', 'yaml', 'yml', 'zip') + +PRIMARY_DOMAIN = 'at.brid.gy' +OTHER_DOMAINS = ( + 'bridgy-at.appspot.com', + 'localhost', +) +DOMAINS = (PRIMARY_DOMAIN,) + OTHER_DOMAINS +# TODO: unify with Bridgy's, Bridgy Fed's +DOMAIN_BLOCKLIST = frozenset(( + 'facebook.com', + 'fb.com', + 't.co', + 'twitter.com', +) + DOMAINS) + +# alias allows unit tests to mock the function +utcnow = datetime.datetime.utcnow diff --git a/config.py b/config.py new file mode 100644 index 0000000..dafdb23 --- /dev/null +++ b/config.py @@ -0,0 +1,21 @@ +"""Flask config. + +https://flask.palletsprojects.com/en/latest/config/ +""" +from oauth_dropins.webutil import appengine_info, util + +# This is primarily for flashed messages, since we don't use session data +# otherwise. +SESSION_COOKIE_SECURE = True +SESSION_COOKIE_HTTPONLY = True +# Change to Lax if/when we add IndieAuth for anything. +SESSION_COOKIE_SAMESITE = 'Strict' + +if appengine_info.DEBUG: + ENV = 'development' + CACHE_TYPE = 'NullCache' + SECRET_KEY = 'sooper seekret' +else: + ENV = 'production' + CACHE_TYPE = 'SimpleCache' + SECRET_KEY = util.read('flask_secret_key') diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..0f69c31 --- /dev/null +++ b/config.yml @@ -0,0 +1,58 @@ +# CircleCI automatically reads this file from our repo and uses it for +# configuration. Docs: +# https://circleci.com/docs/2.1/configuration-reference/ +# https://circleci.com/docs/2.1/sample-config/ +version: 2.1 + +jobs: + build: + docker: + - image: cimg/python:3.9 + + steps: + - checkout + + - restore_cache: + key: venv-1-{{ .Branch }}-{{ checksum "requirements.txt" }} + + - run: + name: Base dependencies + command: | + # google-cloud-sdk: https://cloud.google.com/sdk/docs/install#deb + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates gnupg google-cloud-sdk google-cloud-sdk-datastore-emulator default-jre + + - run: + name: Python dependencies + command: | + pip install -U pip + pip -V + # make sure we install these at head, below + pip uninstall -y granary lexrpc oauth-dropins + pip install -U -r requirements.txt + pip install coverage coveralls + + - run: + name: Build and test + command: | + CLOUDSDK_CORE_PROJECT=bridgy-at gcloud beta emulators datastore start --no-store-on-disk --consistency=1.0 --host-port=localhost:8089 < /dev/null >& /dev/null & + sleep 5s + python -m coverage run --source=. --omit=tests/\* -m unittest discover -v + python -m coverage html -d /tmp/coverage_html + if [ "$COVERALLS_REPO_TOKEN" != "" ]; then coveralls || true; fi + + - save_cache: + key: venv-1-{{ .Branch }}-{{ checksum "requirements.txt" }} + paths: + - /home/circleci/.pyenv + # Ideally we'd cache these, but they need root, and the cimg/python + # Docker image's default user is circleci :/ + # https://github.com/cypress-io/circleci-orb/issues/269 + # + # - /usr/lib/google-cloud-sdk + # - /usr/lib/jvm/java-11-openjdk-amd64 + + - store_artifacts: + path: /tmp/coverage_html diff --git a/dependabot.yml b/dependabot.yml new file mode 100644 index 0000000..55e5ebb --- /dev/null +++ b/dependabot.yml @@ -0,0 +1,8 @@ +# GitHub Dependabot config +# https://docs.github.com/en/github/administering-a-repository/keeping-your-dependencies-updated-automatically +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" diff --git a/feed.py b/feed.py new file mode 100644 index 0000000..771fc0f --- /dev/null +++ b/feed.py @@ -0,0 +1,8 @@ +"""app.bsky.feed.* XRPC methods.""" + +# lexicons/app/bsky/feed/getAuthorFeed.json +# lexicons/app/bsky/feed/getPostThread.json +# lexicons/app/bsky/feed/getRepostedBy.json +# lexicons/app/bsky/feed/getTimeline.json +# lexicons/app/bsky/feed/getVotes.json +# lexicons/app/bsky/feed/setVote.json diff --git a/graph.py b/graph.py new file mode 100644 index 0000000..729a032 --- /dev/null +++ b/graph.py @@ -0,0 +1,7 @@ +"""app.bsky.graph.* XRPC methods.""" + +# lexicons/app/bsky/graph/getAssertions.json +# lexicons/app/bsky/graph/getFollowers.json +# lexicons/app/bsky/graph/getFollows.json +# lexicons/app/bsky/graph/getMembers.json +# lexicons/app/bsky/graph/getMemberships.json diff --git a/models.py b/models.py new file mode 100644 index 0000000..6dd4dd0 --- /dev/null +++ b/models.py @@ -0,0 +1,11 @@ +"""Datastore model classes.""" +import logging + +import requests + +from google.cloud import ndb +from oauth_dropins.webutil.models import StringIdModel +from oauth_dropins.webutil import util +from oauth_dropins.webutil.util import json_dumps, json_loads + +logger = logging.getLogger(__name__) diff --git a/pages.py b/pages.py new file mode 100644 index 0000000..c397a86 --- /dev/null +++ b/pages.py @@ -0,0 +1,146 @@ +"""UI pages.""" +import datetime +import logging + +from flask import redirect, render_template, request +import humanize +from oauth_dropins.webutil import flask_util, logs, util +from oauth_dropins.webutil.flask_util import error, flash, redirect +from oauth_dropins.webutil.util import json_dumps, json_loads + +from app import app, cache +import common +#from models import ... + +PAGE_SIZE = 20 + +logger = logging.getLogger(__name__) + + +@app.route('/') +@flask_util.cached(cache, datetime.timedelta(days=1)) +def front_page(): + """View for the front page.""" + return render_template('index.html') + + +@app.route('/docs') +@flask_util.cached(cache, datetime.timedelta(days=1)) +def docs(): + """View for the docs page.""" + return render_template('docs.html') + + +@app.get('/web-site') +@flask_util.cached(cache, datetime.timedelta(days=1)) +def enter_web_site(): + return render_template('enter_web_site.html') + + +@app.post('/web-site') +def check_web_site(): + url = request.values['url'] + domain = util.domain_from_link(url, minimize=False) + if not domain: + error(f'No domain found in {url}') + + user = User.get_or_create(domain) + try: + user = user.verify() + except BaseException as e: + if util.is_connection_failure(e): + flash(f"Couldn't connect to {url}") + return render_template('enter_web_site.html') + raise + + user.put() + return redirect(f'/user/{user.key.id()}') + + +@app.get(f'/user/') +def user(domain): + user = User.get_by_id(domain) + ... + + +@app.get(f'/user//feed') +def feed(domain): + ... + + +@app.get('/responses') # deprecated +def recent_deprecated(): + return redirect('/recent', code=301) + + +@app.get('/recent') +def recent(): + """Renders recent activities, with links to logs.""" + ... + activities, before, after = fetch_activities(query) + return render_template( + 'recent.html', + show_domains=True, + logs=logs, + util=util, + **locals(), + ) + + +def fetch_page(query, model_class): + """Fetches a page of results from a datastore query. + + Uses the `before` and `after` query params (if provided; should be ISO8601 + timestamps) and the queried model class's `updated` property to identify the + page to fetch. + + Populates a `log_url_path` property on each result entity that points to a + its most recent logged request. + + Args: + query: :class:`ndb.Query` + model_class: ndb model class + + Returns: + (results, new_before, new_after) tuple with: + results: list of query result entities + new_before, new_after: str query param values for `before` and `after` + to fetch the previous and next pages, respectively + """ + ... + + +def fetch_activities(query): + """Fetches a page of Activity entities from a datastore query. + + Wraps :func:`fetch_page` and adds attributes to the returned Activity + entities for rendering in activities.html. + + Args: + query: :class:`ndb.Query` + + Returns: + (results, new_before, new_after) tuple with: + results: list of Activity entities + new_before, new_after: str query param values for `before` and `after` + to fetch the previous and next pages, respectively + """ + ... + + +@app.get('/stats') +def stats(): + def count(kind): + return humanize.intcomma( + KindStat.query(KindStat.kind_name == kind).get().count) + + return render_template( + 'stats.html', + ..., + ) + + +@app.get('/log') +@flask_util.cached(cache, logs.CACHE_TIME) +def log(): + return logs.log() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..85cddf7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,84 @@ +git+https://github.com/snarfed/oauth-dropins.git#egg=oauth_dropins +git+https://github.com/snarfed/granary.git#egg=granary +git+https://github.com/snarfed/lexrpc.git#egg=lexrpc +git+https://github.com/dvska/gdata-python3.git#egg=gdata + +attrs==22.2.0 +beautifulsoup4==4.11.1 +brevity==0.2.17 +cachetools==4.2.4 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.3 +colorama==0.4.6 +Deprecated==1.2.13 +domain2idna==1.12.0 +ecdsa==0.18.0 +extras==1.0.0 +feedgen==0.9.0 +feedparser==6.0.10 +fixtures==4.0.1 +Flask==2.2.2 +Flask-Caching==2.0.1 +flask-gae-static==1.0 +google-api-core==2.11.0 +google-auth==2.15.0 +google-cloud-appengine-logging==1.2.0 +google-cloud-audit-log==0.2.4 +google-cloud-core==2.3.2 +google-cloud-datastore==2.11.0 +google-cloud-logging==3.3.1 +google-cloud-ndb==2.1.0 +googleapis-common-protos==1.57.0 +grpc-google-iam-v1==0.12.4 +grpcio==1.51.1 +gunicorn==20.1.0 +html2text==2020.1.16 +html5lib==1.1 +humanfriendly==10.0 +humanize==4.4.0 +idna==3.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +jsonschema==4.17.3 +lxml==4.9.2 +MarkupSafe==2.1.1 +mf2py==1.1.2 +mf2util==0.5.1 +mox3==1.1.0 +oauthlib==3.2.2 +packaging==22.0 +pbr==5.11.0 +praw==7.6.1 +prawcore==2.3.0 +proto-plus==1.22.1 +protobuf==3.20.3 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycryptodome==3.16.0 +pymemcache==4.0.0 +pyparsing==3.0.9 +pyrsistent==0.19.2 +python-dateutil==2.8.2 +python-tumblpy==1.1.4 +pytz==2022.7 +PyYAML==6.0 +redis==4.4.0 +requests==2.28.1 +requests-oauthlib==1.3.1 +rsa==4.9 +sgmllib3k==1.0.0 +six==1.16.0 +soupsieve==2.3.2.post1 +testtools==2.5.0 +tlslite-ng==0.7.6 +tweepy==4.12.1 +ujson==5.6.0 +update-checker==0.18.0 +urllib3==1.26.13 +webapp2==3.0.0b1 +webencodings==0.5.1 +WebOb==1.8.7 +websocket-client==1.4.2 +Werkzeug==2.2.2 +wrapt==1.14.1 diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..4495f96 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Disallow: /*?* +Disallow: /user/* +Disallow: /recent* diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..19a9194 --- /dev/null +++ b/templates/base.html @@ -0,0 +1,68 @@ + + + + +{% block title %}Bridgy AT{% endblock %} + + + + + + + + + + + + + + + + +{% with messages = get_flashed_messages() %} +{% if messages %} +
+ {% for message in messages %} +

{{ message|safe }}

+ {% endfor %} +
+{% endif %} +{% endwith %} + +
+ +
+ + + +
+ +

+ +{% block content %} +{% endblock %} + + + + +
+ + + + diff --git a/templates/docs.html b/templates/docs.html new file mode 100644 index 0000000..3411e7b --- /dev/null +++ b/templates/docs.html @@ -0,0 +1,419 @@ +{% extends "base.html" %} + +{% block content %} + +
+ +
+ + + +

Personal web site

+ + + +

Fediverse profile via Bridgy AT

+
+ +

+Bridgy AT turns your web site into its own fediverse account, visible in Mastodon and beyond. You can post, reply, like, repost, and follow fediverse accounts by posting on your site with microformats2 and sending webmentions. Bridgy AT translates those posts to fediverse protocols like ActivityPub and OStatus, and sends fediverse interactions back to your site as webmentions. +

+ +

+This isn't syndication or POSSE! You don't need an account on Mastodon or anywhere else. Bridgy AT lets your site act like a first class member of the fediverse. People there will see your posts directly from your site, and vice versa. +

+ +

+Bridgy AT takes some technical know-how to set up, and there are simpler (but less powerful) alternatives. If you just want your site's posts to show up in the fediverse, without any other interactions, consider an RSS or Atom feed bot instead. Or, if you want to cross-post to an existing Mastodon account, try Bridgy. +

+ +
+ + + +
    + +
    +

    Setup

    + +
  • How do I set it up?
    • +
  • +

    +

      +
    1. Your site needs to support SSL. Bridgy AT uses your domain as your identity, so it depends on SSL to prove that you own it.
      2. +
    2. Configure your site to redirect these URL paths to the same paths on https://fed.brid.gy/, including query parameters:
      3. +
      +/.well-known/host-meta
+/.well-known/webfinger
+
      + +

      Here are instructions for a few common web servers:

      + +
        +
      • +

        WordPress (self-hosted): install the Safe Redirect Manager plugin, then add these entries:

        + + /.well-known/host-meta* => https://fed.brid.gy/.well-known/host-meta*
        + /.well-known/webfinger* => https://fed.brid.gy/.well-known/webfinger* +         +
        • + +
      • Known or Drupal: follow the Apache or nginx instructions below. +
        • + +
      • Apache: add this to your .htaccess file:
        +
        RewriteEngine on
+RewriteBase /
+RewriteRule ^.well-known/(host-meta|webfinger).* https://fed.brid.gy/$0  [redirect=302,last]
        +(RewriteEngine on is optional if you already have it earlier in your .htaccess. RewriteBase / is optional if you don't have any other RewriteBase directives, or if you put this RewriteRule inside an existing RewriteBase / section.) +
        • + +
      • nginx: add this to your nginx.conf file, in the server section:
        + 
        rewrite ^/\.well-known/(host-meta|webfinger).* https://fed.brid.gy$request_uri? redirect;
        +
        • + +
      • Netlify: add this to your netlify.toml file. + 
        +[[redirects]]
+  from = "/.well-known/host-meta*"
+  to = "https://fed.brid.gy/.well-known/host-meta:splat"
+  status = 302
+[[redirects]]
+  from = "/.well-known/webfinger*"
+  to = "https://fed.brid.gy/.well-known/webfinger"
+  status = 302
+
        +
        • + + + + +
      + +
    3. Add webmention support to your site. This is strongly recommended, but technically optional. You don't have to automate the webmentions to Bridgy AT to federate your posts, and you don't have to accept the inbound webmentions that Bridgy AT sends, but you'll have a much better experience if you do. Check out the IndieWeb wiki for instructions for your web server.
      4. +
    +
    • + +
  • How do I set up my profile?
    • +
  • +

    +Your site's fediverse profile comes from the microformats2 representative h-card on your site's home page. Here's a minimal example to set your name and a profile picture: + +

    +<span class="h-card">
+  <a rel="me" href="/">Alice Foo</a>
+  <img class="u-photo" src="/me.jpg" />
+</span>
+
    +

    + +

    If you want to set a header image, add a u-featured image to your h-card, eg: + +

    +<img class="u-featured" src="/my-header.png" />
+
    +

    + +

    By default, your fediverse address will be @yourdomain.com@yourdomain.com. Many services (eg Mastodon) default to only showing the username, so this generally shows up as just @yourdomain.com in posts, and the full address appears on hover.

    + +

    We recommend this for simplicity and predictability, for everyone else as well as you, but if you want a different username, you can set it by adding an acct: u-url link inside your h-card with username@yourdomain.com, eg: + +

    +<a class="u-url" href="acct:alice@yourdomain.com"></a>
+
    +
    • + +
  • Where's my user page and dashboard?
    • +
  • +

    +Enter your domain here to see your user page. It shows your site's current status, recent interactions, remote follow UI, and links to your timeline feeds in various formats. +

    +
    • + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    Usage

    + +
  • How do I post to the fediverse?
    • +
  • +

    +Create a post with the h-entry microformat on your web site. Many web servers include this or compatible microformats automatically. The post can be a note, article, like, repost, reply, or follow. For example: + +

    <div class="h-entry">
+  <p class="e-content">Two naked tags walk into a bar. The bartender exclaims, "Hey, you can't come in here without microformats, this is a classy joint!"</p>
+  <a class="u-bridgy-at" href="https://fed.brid.gy/"></a>
+</div>
+
    +

    + +

    Basic HTML formatting like links, bold, and italics are generally preserved and visible in the fediverse, but specifics vary from site to site. +

    + +

    Then, include a link (optionally blank) to https://fed.brid.gy/ in that post and send Bridgy AT a webmention. That webmention will trigger Bridgy AT to forward your post into the fediverse. Your web server may send the webmention automatically if it supports them, or you can send it manually. +

    + +

    (The u-bridgy-at class isn't strictly necessary, but it's useful in some cases to prevent microformats2 parsers from interpreting the link as an implied u-url.) +

    +
    • + +
  • Which of my posts will show up in the fediverse?
    • +
  • +

    Only the ones you explicitly trigger with a webmention. Bridgy AT doesn't automatically create posts in the fediverse based on your site's Atom feed, HTML, or anything else. It only create posts in the fediverse on an opt in basis, per post, via webmention. +

    +
    • + +
  • How does it decide which parts of my posts to include?
    • +
  • +

    Magic! Most major blog engines and CMSes are supported out of the box, no setup necessary. Bridgy AT looks for microformats in your HTML, first the microformats2 e-content class and then the legacy entry-content class. It also understands more advanced microformats2 classes like in-reply-to, u-like-of, u-repost-of, and u-photo. +

    + +

    Bridgy AT sends the full contents of all posts, specifically everything inside e-content, to the fediverse. However, not all fediverse apps currently show the full contents of all posts. +

    + +

    For example, text-based posts fall into two broad buckets: short notes, eg tweets and toots, and longer articles, eg blog posts. In the IndieWeb, we differentiate based on whether the post has a title: articles generally have titles, notes don't. +

    + +

    Mastodon currently shows the full text of notes, but for articles, it only shows their titles and a link to the full article. This is because Mastodon and most other fediverse apps are designed primarily for smaller notes, not longer articles. +

    • + + +
  • +

    These can happen for a couple reasons. For articles, this is expected behavior, as described above. The link is a Bridgy AT URL that redirects to the original post on your web site. This is because Mastodon requires ActivityPub (ie fediverse) object URLs to use the same domain that serves them, which in this case is fed.brid.gy. We know it's awkward; sorry for the ugliness! +

    + +

    Otherwise, this may be the invisible fed.brid.gy link that's required to trigger Bridgy AT. Mastodon will show a preview of links even if their text is blank, so if your link is inside your e-content microformats2 element, that's probably what's happening. You can prevent that by moving it outside of e-content. It can go anywhere in your HTML! +

    +
    • + +
  • How do I reply to a fediverse post?
    • +
  • +

    +Put the reply in a new post on your web site, and include a link to the fediverse post you're replying to with class u-in-reply-to, as if you were publishing a normal IndieWeb reply. For example: + +

    <div class="h-entry">
+  <p class="e-content">Highly entertaining. Please subscribe me to your newsletter.</p>
+  <a class="u-in-reply-to" href="https://indieweb.social/@tchambers/109243684867780200"></a>
+  <a class="u-bridgy-at" href="https://fed.brid.gy/"></a>
+</div>
+
    +

    +
    • + +
  • How do I favorite (aka like) or boost (aka repost) a fediverse post?
    • +
  • +

    Favoriting and boosting are almost exactly the same as replying. The only difference is that you use u-like-of for a favorite/like or u-repost-of for a boost/repost. + +

    +<a class="u-like-of" href="https://octodon.social/@cwebber/109405439825087368"></a>
+
    + +
    +<a class="u-repost-of" href="https://prodromou.pub/@evan/109390803478257847"></a>
+
    +

    +
    • + +
  • How do I follow someone?
    • +
  • +

    +Post an IndieWeb follow on your site with u-follow-of microformats2, then send a webmention to Bridgy AT. Your site may do that automatically if it supports webmentions. For example: +

    + +
    <div class="h-entry">
+  I'm now following <a class="u-follow-of" href="https://octodon.social/@cwebber">@cwebber@octodon.social</a>!
+  <a class="u-bridgy-at" href="https://fed.brid.gy/"></a>
+</div>
+
    +

    +
    • + +
  • How do I include an image in a post?
    • +
  • +

    +Use <img class="u-photo"> for the image in your post. For example: + +

    +<img class="u-photo" src="/full_glass.jpg" />
+I love scotch. Scotchy scotchy scotch.
+
    +

    +
    • + +
  • How do I edit an existing post?
    • +
  • +

    Edit the post on your web site, then send another webmention to Bridgy AT for it. Bridgy AT will refetch the post and send an Update activity for it to the fediverse. +

    +
    • + +
  • Can I publish just one part of a page?
    • +
  • +

    If that HTML element has its own id, then sure! Just put the id in the fragment of the URL that you publish. For example, to publish the bar post here:

    +
    <div id="a" class="h-entry">foo</div>
+<div id="b" class="h-entry">bar</div>
+<div id="c" class="h-entry">baz</div>
+
    +

    ...just add the id to your page's URL in a fragment, e.g. http://site/post#b here.

    +
    • + +
  • How do fediverse replies, likes, and other interactions show up on my site?
    • +
  • +

    +To receive likes, reposts, replies, @-mentions, and follows from the fediverse, just make sure your site accepts webmentions! Bridgy AT translates those interactions and sends them to your site as webmentions. The source URL will usually be a proxy page on fed.brid.gy. For best results, make sure your webmention handler detects and handles u-url links. +

    +
    • + +
  • How do I read my fediverse timeline/feed?
    • +
  • +

    Your user page has links to your fediverse timeline/feed, ie posts from people you follow, in HTML, Atom, and RSS formats. Add them to your feed reader or read them in your browser! +

    +
    • + +
  • How can people on the fediverse find and follow me?
    • +
  • +

    They can search for your web site in any Mastodon instance! Often you can just enter your domain, eg yourdomain.com, in any Mastodon search box. If that doesn't work, try your full fediverse address, eg @yourdomain.com@yourdomain.com. This can be finicky now and then, but it usually works. +

    + +

    Your user page also has a "remote follow" form that lets people enter their fediverse address and follow you directly. +

    +
    • + +
  • I tried it, and it didn't work!
    • +
  • +

    Check out your user page! It detects and describes common problems with your setup, and it shows your recent interactions and detailed logs. +

    • + + +
    +

    About

    + +
  • Who are you? Why did you make this?
    • +
  • +

    +I'm Ryan Barrett. I'm just a guy who likes the web and owning my data. +

    +
    • + +
  • How much does it cost?
    • +
  • +

    Nothing! Bridgy AT is small, and it doesn't cost much to run. We don't need donations, promise. +

    +

    If you really want to contribute, file an issue or send a pull request, or donate to the IndieWeb! +

    • + +
  • What do you do with my data?
    • +
  • +

    Nothing! Bridgy AT isn't a business, and never will be, so we don't have the same motivations to abuse your data that other services might. More concretely, Bridgy AT won't ever send you email, it stores as little of your PII (personally identifiable information) as possible, and it never has access to any of your passwords. +

    +
    • + +
  • How long has this been around?
    • +
  • +

    I started thinking about bridging federated social networks and peer to peer networks when I discovered them in the early 2000s. I started talking about bridging them to the IndieWeb in 2016, led a session on it at IndieWeb Summit in July 2017, wrote up concrete designs soon after, started working on Bridgy AT in August 2017, and launched it on October 22, 2017. +

    • + +
  • What are the terms of service?
    • +
  • +

    Bridgy AT's terms of service are very simple. You agree not to deliberately attack, breach, or otherwise harm the service. If you manage to access private keys or other private data, you agree to report the vulnerability and not use or disclose that data. +

    +

    Otherwise, you may use the service for any purpose you see fit. However, we may terminate or block your access for any reason, or no reason at all. (We've never done this, and we expect we never will. Just playing it safe.) +

    +

    Do you an administer an instance or other service that Bridgy AT interacts with? If you have any concerns or questions, feel free to file an issue! +

    +
    • + +
  • I found a bug! I have a feature request!
    • +
  • +

    Great! Please file it in GitHub. Thank you! +

    +
    • + +
  • I found a security vulnerability!
    • +
  • +

    Oof. Thank you for reporting it! Please send details to security@brid.gy. We may provide monetary awards for reports of significant vulnerabilities, eg reading or modifying stored access tokens, if you follow these rules:

    +
      +
    • Vulnerabilities must be in the application itself, not unrelated services like email (eg SPF/DKIM/DMARC).
      • +
    • Out of scope: rate limiting, XSS/CSRF attacks (Bridgy AT has no authenticated sessions or private data accessible to users), /admin/* pages. +
    • Public user data is intentionally public. That's not a vulnerability.
      • +
    • No automated fuzzing, DoSes, or other high volume traffic. We block this traffic, and it will disqualify you from any possible award.
      • +
    +

    Otherwise, the code is open source, feel free to try to break in, let us know if you succeed!

    +
    • + +
+
+ +{% endblock %} diff --git a/templates/enter_web_site.html b/templates/enter_web_site.html new file mode 100644 index 0000000..53f1d7b --- /dev/null +++ b/templates/enter_web_site.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block content %} + +

What's your web site?

+ +
+
+ + +
+
+ +{% endblock %} diff --git a/templates/feed.html b/templates/feed.html new file mode 100644 index 0000000..345b4c5 --- /dev/null +++ b/templates/feed.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} + +{% block title %}{{ domain }}'s feed - Bridgy AT{% endblock %} + +{% block content %} + +{% include "user_addresses.html" %} + +
Feed
+ + +
+{% for e in entries %} + {{ e|safe }} +{% else %} + Nothing yet. Follow more people, check back soon! +{% endfor %} +
+ +{% endblock %} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..cb149e9 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} + +{% block content %} + +
+ What do you want to do? +
+ +
+ +
+ +

Cross-post to a Mastodon account:
+@you@mastodon.server

+
      
+    /
+🌐 —  — 
+    \
+      
+
+

You have a Mastodon account. You want to post on your web site, copy those posts to that account, and send responses back to your site.

+ +
+ +
+ +

Connect directly to the fediverse:
+@yoursite.com

+
   
+\ /
+ — 🌐 — 
+/ \
+   
+
+

Your web site will be its own fediverse account, including its posts. Its username and instance will be your domain. This takes a bit of technical known-how.

+ +
+
+ +
+
+ ...or create an account on micro.blog or Mastodon to try out the fediverse first. Ask us more in chat! +
+
+ + + +{% endblock %} diff --git a/templates/paging.html b/templates/paging.html new file mode 100644 index 0000000..767a26a --- /dev/null +++ b/templates/paging.html @@ -0,0 +1,13 @@ +
+
+ {% if after %} + ← Newer + {% endif %} +
+ +
+ {% if before %} + Older → + {% endif %} +
+
diff --git a/templates/stats.html b/templates/stats.html new file mode 100644 index 0000000..61a234a --- /dev/null +++ b/templates/stats.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} + +{% block title %}Stats - Bridgy AT{% endblock %} + +{% block content %} + +

Stats

+
    +
  • Launched ...? +
  • {{ users }} users
    • +
  • {{ followers }} fediverse followers
    • +
  • {{ activities }} activities handled
    • +
+ + +{% endblock %} diff --git a/templates/user.html b/templates/user.html new file mode 100644 index 0000000..11bc85b --- /dev/null +++ b/templates/user.html @@ -0,0 +1,68 @@ +{% extends "base.html" %} + +{% block title %}{{ domain }} - Bridgy AT{% endblock %} + +{% block content %} + +{% if user.has_redirects == False %} +
+
+ Next step: + + add the .well-known redirects. + + + +
+ +{% if user.redirects_error %} +
+ {{ user.redirects_error|safe }} +
+{% endif %} +
+{% endif %} + +{% if user.has_hcard == False %} +
+
+ Next step: + + add a representative h-card. + + + +
+
+{% endif %} + +{% include "user_addresses.html" %} + +
+ {{ followers }} follower{% if followers != '1' %}s{% endif %} + · following {{ following }} + · HTML + · Atom + · RSS + · Webfinger + · ActivityPub +
+ +
+
+

+ + + + +

+
+
+ + + +{% include "activities.html" %} + +{% endblock %} diff --git a/templates/user_addresses.html b/templates/user_addresses.html new file mode 100644 index 0000000..43478ac --- /dev/null +++ b/templates/user_addresses.html @@ -0,0 +1,16 @@ +
+
+ {{ user.user_page_link()|safe }} + · + + + + {{ user.address() }} + + + · + + 🌐 {{ domain }} + +
+
diff --git a/templates/user_not_found.html b/templates/user_not_found.html new file mode 100644 index 0000000..c1447b6 --- /dev/null +++ b/templates/user_not_found.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block title %}User not found - Bridgy AT{% endblock %} + +{% block content %} + +
+
+ {{ domain }} is not (yet!) a Bridgy AT user. +
+
+
+ +{% endblock %} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..aaf00d9 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +# configure logging +import oauth_dropins.webutil.tests diff --git a/tests/test_actor.py b/tests/test_actor.py new file mode 100644 index 0000000..6ed633a --- /dev/null +++ b/tests/test_actor.py @@ -0,0 +1,8 @@ +"""Unit tests for actor.py.""" +from . import testutil + + +class ActorTest(testutil.TestCase): + + def test_noop(self): + pass diff --git a/tests/test_feed.py b/tests/test_feed.py new file mode 100644 index 0000000..1bb786a --- /dev/null +++ b/tests/test_feed.py @@ -0,0 +1 @@ +"""Unit tests for feed.py.""" diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 0000000..1fc1a00 --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1 @@ +"""Unit tests for graph.py.""" diff --git a/tests/testutil.py b/tests/testutil.py new file mode 100644 index 0000000..0692766 --- /dev/null +++ b/tests/testutil.py @@ -0,0 +1,33 @@ +"""Common test utility code.""" +import datetime +import unittest + +import requests + +from app import app, cache +import common +from oauth_dropins.webutil import testutil, util +from oauth_dropins.webutil.appengine_config import ndb_client +from oauth_dropins.webutil.util import json_dumps, json_loads + +NOW = datetime.datetime(2022, 12, 24, 22, 29, 19) + + +class TestCase(unittest.TestCase, testutil.Asserts): + maxDiff = None + + def setUp(self): + super().setUp() + app.testing = True + cache.clear() + self.client = app.test_client() + common.utcnow = lambda: NOW + + # clear datastore + requests.post('http://%s/reset' % ndb_client.host) + self.ndb_context = ndb_client.context() + self.ndb_context.__enter__() + + def tearDown(self): + self.ndb_context.__exit__(None, None, None) + super().tearDown() diff --git a/workflows/auto-merge-dependabot.yaml b/workflows/auto-merge-dependabot.yaml new file mode 100644 index 0000000..08f1b1e --- /dev/null +++ b/workflows/auto-merge-dependabot.yaml @@ -0,0 +1,37 @@ +# Auto-merge Dependabot PRs that upgrade patch or minor versions if CI passes +# Copied from https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request +# Also see https://github.com/dependabot/fetch-metadata + +name: Dependabot auto-merge +on: + pull_request: + branches: main + workflow_dispatch: + +permissions: + pull-requests: write + contents: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: github.actor == 'dependabot[bot]' + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1.1.1 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Enable auto-merge for Dependabot PRs + if: > + ! contains(steps.metadata.outputs.dependency-names, 'tlslite-ng') && + steps.metadata.outputs.update-type != 'version-update:semver-major' + run: gh pr merge --auto --rebase "$PR_URL" + + - name: "Warn that we won't auto-merge major version updates" + if: steps.metadata.outputs.update-type == 'version-update:semver-major' + run: gh pr comment "$PR_URL" -b "Looks like a major version upgrade! Skipping auto-merge." diff --git a/workflows/codeql-analysis.yml b/workflows/codeql-analysis.yml new file mode 100644 index 0000000..e160cf5 --- /dev/null +++ b/workflows/codeql-analysis.yml @@ -0,0 +1,64 @@ +name: "CodeQL" + +on: + push: + branches: [main] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '0 19 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python', 'javascript'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # Command-line programs to run using the OS shell. https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 + + - name: DefenseCode ThunderScan Action + uses: defensecode/thunderscan-action@v1.0 diff --git a/workflows/dependency-review.yaml b/workflows/dependency-review.yaml new file mode 100644 index 0000000..90cc888 --- /dev/null +++ b/workflows/dependency-review.yaml @@ -0,0 +1,17 @@ +# Prevents merging dependency versions w/vulnerabilities +# https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review +# https://github.com/actions/dependency-review-action#installation= +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v3 + - name: 'Dependency Review' + uses: actions/dependency-review-action@v1