2023-03-08 21:10:41 +00:00
""" Common test utility code. """
2017-10-20 14:13:04 +00:00
import copy
2023-04-30 19:20:58 +00:00
from datetime import datetime
import logging
2023-04-20 22:04:00 +00:00
import random
2023-06-07 23:40:31 +00:00
import re
2017-08-24 06:24:47 +00:00
import unittest
2019-12-26 06:20:57 +00:00
from unittest . mock import ANY , call
2023-06-09 04:21:40 +00:00
import warnings
2017-08-24 06:24:47 +00:00
2023-05-06 21:37:23 +00:00
import arroba . util
from arroba . util import datetime_to_tid
2023-06-09 04:21:40 +00:00
from bs4 import MarkupResemblesLocatorWarning
2023-04-20 22:04:00 +00:00
import dag_cbor . random
2023-03-20 21:28:14 +00:00
from flask import g
2023-03-27 21:12:06 +00:00
from google . cloud import ndb
2023-01-07 17:18:11 +00:00
from granary import as2
2023-02-09 04:22:16 +00:00
from granary . tests . test_as1 import (
COMMENT ,
MENTION ,
NOTE ,
)
2017-10-20 14:13:04 +00:00
from oauth_dropins . webutil import testutil , util
2019-12-26 06:20:57 +00:00
from oauth_dropins . webutil . appengine_config import ndb_client
2023-01-07 17:18:11 +00:00
from oauth_dropins . webutil . testutil import requests_response
2019-12-26 06:20:57 +00:00
import requests
2017-08-24 06:24:47 +00:00
2023-07-13 21:19:01 +00:00
# other modules are imported _after_ Fake etc classes is defined so that it's in
2023-05-30 23:36:18 +00:00
# PROTOCOLS when URL routes are registered.
2023-05-26 23:07:36 +00:00
import models
2023-03-27 21:12:06 +00:00
from models import Object , PROTOCOLS , Target , User
2023-03-08 21:10:41 +00:00
import protocol
2023-04-19 00:17:48 +00:00
2023-03-27 21:12:06 +00:00
logger = logging . getLogger ( __name__ )
2023-03-14 00:23:57 +00:00
2023-05-26 23:07:36 +00:00
class Fake ( User , protocol . Protocol ) :
2023-06-11 15:14:17 +00:00
ABBREV = ' fa '
2023-03-14 00:23:57 +00:00
2023-06-27 16:48:47 +00:00
# maps string ids to dict AS1 objects that can be fetched
fetchable = { }
2023-03-27 21:12:06 +00:00
# in-order list of (Object, str URL)
sent = [ ]
# in-order list of ids
fetched = [ ]
2023-06-01 01:34:33 +00:00
def web_url ( self ) :
2023-06-15 17:52:11 +00:00
return self . key . id ( )
2023-06-01 01:34:33 +00:00
2023-05-31 17:47:09 +00:00
def ap_address ( self ) :
2023-06-01 01:34:33 +00:00
return f ' @ { self . key . id ( ) } @fake '
2023-05-31 17:47:09 +00:00
def ap_actor ( self , rest = None ) :
2023-06-12 22:37:17 +00:00
return f ' http://bf/fake/ { self . key . id ( ) } /ap ' + ( f ' / { rest } ' if rest else ' ' )
2023-05-31 17:47:09 +00:00
2023-06-13 20:17:11 +00:00
@classmethod
def owns_id ( cls , id ) :
2023-06-15 17:52:11 +00:00
if id . startswith ( ' nope ' ) :
return False
2023-06-27 16:48:47 +00:00
return id . startswith ( ' fake: ' ) or id in cls . fetchable
2023-06-13 20:17:11 +00:00
2023-03-14 00:23:57 +00:00
@classmethod
2023-03-21 02:25:05 +00:00
def send ( cls , obj , url , log_data = True ) :
2023-05-26 23:07:36 +00:00
logger . info ( f ' Fake.send { url } ' )
2023-04-03 14:53:15 +00:00
cls . sent . append ( ( obj , url ) )
2023-06-27 18:39:57 +00:00
return True
2023-03-14 00:23:57 +00:00
@classmethod
2023-06-12 22:50:47 +00:00
def fetch ( cls , obj , * * kwargs ) :
2023-04-03 14:53:15 +00:00
id = obj . key . id ( )
2023-06-29 05:46:53 +00:00
logger . info ( f ' Fake.fetch { id } ' )
2023-03-27 21:12:06 +00:00
cls . fetched . append ( id )
2023-06-27 16:48:47 +00:00
if id in cls . fetchable :
obj . our_as1 = cls . fetchable [ id ]
2023-07-14 19:45:47 +00:00
return True
2023-03-27 21:12:06 +00:00
2023-07-14 19:45:47 +00:00
return False
2023-03-08 21:10:41 +00:00
2023-05-24 04:30:57 +00:00
@classmethod
def serve ( cls , obj ) :
2023-05-26 23:07:36 +00:00
logger . info ( f ' Fake.load { obj . key . id ( ) } ' )
return ( f ' Fake object { obj . key . id ( ) } ' ,
2023-05-24 04:30:57 +00:00
{ ' Accept ' : ' fake/protocol ' } )
2023-06-17 21:12:43 +00:00
@classmethod
def target_for ( cls , obj , shared = False ) :
2023-06-21 14:22:03 +00:00
assert obj . source_protocol in ( cls . LABEL , cls . ABBREV , ' ui ' , None )
2023-06-27 03:22:06 +00:00
return ' shared:target ' if shared else f ' { obj . key . id ( ) } :target '
2023-06-17 21:12:43 +00:00
2023-06-30 18:33:03 +00:00
@classmethod
2023-07-02 05:40:42 +00:00
def receive ( cls , our_as1 ) :
assert isinstance ( our_as1 , dict )
return super ( ) . receive ( Object ( id = our_as1 [ ' id ' ] , our_as1 = our_as1 ) )
2023-06-30 18:33:03 +00:00
2023-03-10 23:13:45 +00:00
2023-07-13 21:19:01 +00:00
class OtherFake ( Fake ) :
""" Different class because the same-protocol check special cases Fake.
Used in ProtocolTest . test_skip_same_protocol
"""
ABBREV = ' other '
@classmethod
def owns_id ( cls , id ) :
return id . startswith ( ' other: ' )
2023-05-26 23:07:36 +00:00
# used in TestCase.make_user() to reuse keys across Users since they're
# expensive to generate
requests . post ( f ' http:// { ndb_client . host } /reset ' )
with ndb_client . context ( ) :
2023-06-15 17:52:11 +00:00
global_user = Fake . get_or_create ( ' fake:user ' )
2023-05-26 23:07:36 +00:00
2023-05-30 23:36:18 +00:00
# import other modules that register Flask handlers *after* Fake is defined
2023-05-26 23:07:36 +00:00
models . reset_protocol_properties ( )
2023-05-30 23:36:18 +00:00
import app
2023-05-31 20:17:17 +00:00
from activitypub import ActivityPub , CONNEG_HEADERS_AS2_HTML
2023-05-30 23:36:18 +00:00
import common
from web import Web
from flask_app import app , cache , init_globals
2023-05-26 23:07:36 +00:00
2017-10-10 14:42:10 +00:00
class TestCase ( unittest . TestCase , testutil . Asserts ) :
2017-08-24 06:24:47 +00:00
maxDiff = None
def setUp ( self ) :
2021-07-11 23:30:14 +00:00
super ( ) . setUp ( )
2023-03-27 21:12:06 +00:00
2021-08-18 14:59:52 +00:00
app . testing = True
cache . clear ( )
2023-03-08 21:10:41 +00:00
protocol . seen_ids . clear ( )
2023-04-03 03:36:23 +00:00
protocol . objects_cache . clear ( )
2023-03-11 06:24:58 +00:00
common . webmention_discover . cache . clear ( )
2023-02-09 04:22:16 +00:00
2023-06-27 16:48:47 +00:00
Fake . fetchable = { }
2023-05-26 23:07:36 +00:00
Fake . sent = [ ]
Fake . fetched = [ ]
2023-03-27 21:12:06 +00:00
2023-06-12 22:37:17 +00:00
common . OTHER_DOMAINS + = ( ' fake.brid.gy ' , )
common . DOMAINS + = ( ' fake.brid.gy ' , )
2023-04-30 19:20:58 +00:00
# make random test data deterministic
2023-05-06 21:37:23 +00:00
arroba . util . _clockid = 17
2023-04-30 19:20:58 +00:00
random . seed ( 1234567890 )
dag_cbor . random . set_options ( seed = 1234567890 )
2021-08-18 14:59:52 +00:00
self . client = app . test_client ( )
2023-02-09 04:22:16 +00:00
self . client . __enter__ ( )
2019-12-26 06:20:57 +00:00
# clear datastore
2023-01-24 20:17:24 +00:00
requests . post ( f ' http:// { ndb_client . host } /reset ' )
2023-06-22 21:27:02 +00:00
# disable in-memory cache
# (also in flask_app.py)
# https://github.com/googleapis/python-ndb/issues/888
self . ndb_context = ndb_client . context ( cache_policy = lambda key : False )
2019-12-26 06:20:57 +00:00
self . ndb_context . __enter__ ( )
2017-08-24 06:24:47 +00:00
2023-01-13 03:43:12 +00:00
util . now = lambda * * kwargs : testutil . NOW
2023-06-16 04:22:20 +00:00
# used in make_user()
self . last_make_user_id = 1
2023-01-13 03:43:12 +00:00
2023-05-23 06:09:36 +00:00
self . app_context = app . app_context ( )
self . app_context . push ( )
init_globals ( )
self . request_context = app . test_request_context ( ' / ' )
2023-06-15 22:09:03 +00:00
self . request_context . push ( )
2023-05-23 06:09:36 +00:00
2023-06-09 04:21:40 +00:00
# suppress a few warnings
# local/lib/python3.9/site-packages/bs4/__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.
warnings . filterwarnings ( ' ignore ' , category = MarkupResemblesLocatorWarning )
2017-08-24 06:24:47 +00:00
def tearDown ( self ) :
2023-05-23 06:09:36 +00:00
self . app_context . pop ( )
2019-12-26 06:20:57 +00:00
self . ndb_context . __exit__ ( None , None , None )
2023-02-09 04:22:16 +00:00
self . client . __exit__ ( None , None , None )
2021-07-11 23:30:14 +00:00
super ( ) . tearDown ( )
2017-10-20 14:13:04 +00:00
2023-06-15 22:09:03 +00:00
# this breaks if it's before super().tearDown(). why?!
self . request_context . pop ( )
2023-06-07 23:40:31 +00:00
def run ( self , result = None ) :
""" Override to hide stdlib and virtualenv lines in tracebacks.
https : / / docs . python . org / 3.9 / library / unittest . html #unittest.TestCase.run
https : / / docs . python . org / 3.9 / library / unittest . html #unittest.TestResult
"""
result = super ( ) . run ( result = result )
def prune ( results ) :
return [
2023-07-10 18:37:40 +00:00
( tc , re . sub ( r ' \ n File " .+/(local|.venv|oauth-dropins|Python.framework)/.+ \ n.+ \ n ' ,
2023-06-07 23:40:31 +00:00
' \n ' , tb ) )
for tc , tb in results ]
result . errors = prune ( result . errors )
result . failures = prune ( result . failures )
return result
2023-06-10 22:07:26 +00:00
# TODO: switch default to Fake, start using that more
2023-06-16 04:22:20 +00:00
def make_user ( self , id , cls = Web , * * kwargs ) :
2023-03-10 23:13:45 +00:00
""" Reuse RSA key across Users because generating it is expensive. """
2023-06-16 04:22:20 +00:00
obj_key = None
2023-06-27 03:22:06 +00:00
obj_as2 = kwargs . pop ( ' obj_as2 ' , None ) or { }
obj_mf2 = kwargs . pop ( ' obj_mf2 ' , None ) or { }
obj_id = kwargs . pop ( ' obj_id ' , None )
if not obj_id :
obj_id = ( obj_as2 . get ( ' id ' )
or util . get_url ( obj_mf2 , ' properties ' )
or str ( self . last_make_user_id ) )
2023-06-16 04:22:20 +00:00
self . last_make_user_id + = 1
2023-06-27 03:22:06 +00:00
obj_key = Object ( id = obj_id , as2 = obj_as2 , mf2 = obj_mf2 ) . put ( )
2023-06-16 04:22:20 +00:00
2023-05-30 23:36:18 +00:00
user = cls ( id = id ,
2023-05-30 03:16:15 +00:00
direct = True ,
2023-05-26 23:07:36 +00:00
mod = global_user . mod ,
public_exponent = global_user . public_exponent ,
private_exponent = global_user . private_exponent ,
p256_key = global_user . p256_key ,
2023-06-16 04:22:20 +00:00
obj_key = obj_key ,
2023-05-26 23:07:36 +00:00
* * kwargs )
2023-03-10 23:13:45 +00:00
user . put ( )
return user
2023-05-23 06:09:36 +00:00
def add_objects ( self ) :
2023-07-16 21:06:03 +00:00
user = ndb . Key ( Web , ' user.com ' )
2023-06-29 05:46:53 +00:00
# post
2023-07-16 21:06:03 +00:00
self . store_object ( id = ' a ' ,
users = [ user ] ,
notify = [ user ] ,
feed = [ user ] ,
2023-06-29 05:46:53 +00:00
as2 = as2 . from_as1 ( NOTE ) )
# different domain
2023-07-16 21:06:03 +00:00
nope = ndb . Key ( Web , ' nope.org ' )
self . store_object ( id = ' b ' ,
notify = [ nope ] ,
feed = [ nope ] ,
2023-06-29 05:46:53 +00:00
as2 = as2 . from_as1 ( MENTION ) )
# reply
2023-07-16 21:06:03 +00:00
self . store_object ( id = ' d ' ,
notify = [ user ] ,
feed = [ user ] ,
2023-06-29 05:46:53 +00:00
as2 = as2 . from_as1 ( COMMENT ) )
# not feed/notif
2023-07-16 21:06:03 +00:00
self . store_object ( id = ' e ' , users = [ user ] , as2 = as2 . from_as1 ( NOTE ) )
2023-06-29 05:46:53 +00:00
# deleted
2023-07-16 21:06:03 +00:00
self . store_object ( id = ' f ' ,
notify = [ user ] ,
feed = [ user ] ,
2023-06-29 05:46:53 +00:00
as2 = as2 . from_as1 ( NOTE ) , deleted = True )
@staticmethod
def store_object ( * * kwargs ) :
obj = Object ( * * kwargs )
obj . put ( )
2023-07-24 21:07:44 +00:00
protocol . objects_cache . pop ( obj . key . id ( ) , None )
2023-06-29 05:46:53 +00:00
return obj
2023-02-09 04:22:16 +00:00
2023-04-30 19:20:58 +00:00
@staticmethod
def random_keys_and_cids ( num ) :
def tid ( ) :
ms = random . randint ( datetime ( 2020 , 1 , 1 ) . timestamp ( ) * 1000 ,
datetime ( 2024 , 1 , 1 ) . timestamp ( ) * 1000 )
return datetime_to_tid ( datetime . fromtimestamp ( float ( ms ) / 1000 ) )
return [ ( f ' com.example.record/ { tid ( ) } ' , cid )
for cid in dag_cbor . random . rand_cid ( num ) ]
def random_tid ( num ) :
ms = random . randint ( datetime ( 2020 , 1 , 1 ) . timestamp ( ) * 1000 ,
datetime ( 2024 , 1 , 1 ) . timestamp ( ) * 1000 )
tid = datetime_to_tid ( datetime . fromtimestamp ( float ( ms ) / 1000 ) )
return f ' com.example.record/ { tid } '
2023-05-30 23:36:18 +00:00
def get_as2 ( self , * args , * * kwargs ) :
2023-05-31 20:17:17 +00:00
kwargs . setdefault ( ' headers ' , { } ) [ ' Accept ' ] = CONNEG_HEADERS_AS2_HTML
2023-05-30 23:36:18 +00:00
return self . client . get ( * args , * * kwargs )
2023-06-05 20:20:07 +00:00
@classmethod
def req ( cls , url , * * kwargs ) :
2017-10-20 14:13:04 +00:00
""" Returns a mock requests call. """
2022-11-24 16:20:04 +00:00
kwargs . setdefault ( ' headers ' , { } ) . update ( {
' User-Agent ' : util . user_agent ,
} )
2017-10-20 14:13:04 +00:00
kwargs . setdefault ( ' timeout ' , util . HTTP_TIMEOUT )
2019-10-04 04:08:26 +00:00
kwargs . setdefault ( ' stream ' , True )
2017-10-20 14:13:04 +00:00
return call ( url , * * kwargs )
2022-03-17 04:11:09 +00:00
2023-06-05 20:20:07 +00:00
@classmethod
def as2_req ( cls , url , * * kwargs ) :
2022-11-24 16:20:04 +00:00
headers = {
2023-01-16 21:00:38 +00:00
' Date ' : ' Sun, 02 Jan 2022 03:04:05 GMT ' ,
2022-11-24 16:20:04 +00:00
' Host ' : util . domain_from_link ( url , minimize = False ) ,
2022-11-24 17:39:01 +00:00
' Content-Type ' : ' application/activity+json ' ,
' Digest ' : ANY ,
2023-05-31 20:17:17 +00:00
* * CONNEG_HEADERS_AS2_HTML ,
2022-11-24 16:20:04 +00:00
* * kwargs . pop ( ' headers ' , { } ) ,
}
2023-06-05 20:20:07 +00:00
return cls . req ( url , data = None , auth = ANY , headers = headers ,
allow_redirects = False , * * kwargs )
2023-02-07 03:23:25 +00:00
2023-06-05 20:20:07 +00:00
@classmethod
def as2_resp ( cls , obj ) :
2023-01-07 17:18:11 +00:00
return requests_response ( obj , content_type = as2 . CONTENT_TYPE )
2022-03-17 04:11:09 +00:00
def assert_req ( self , mock , url , * * kwargs ) :
""" Checks a mock requests call. """
2022-11-24 17:39:01 +00:00
kwargs . setdefault ( ' headers ' , { } ) . setdefault (
' User-Agent ' , ' Bridgy Fed (https://fed.brid.gy/) ' )
2022-03-17 04:11:09 +00:00
kwargs . setdefault ( ' stream ' , True )
kwargs . setdefault ( ' timeout ' , util . HTTP_TIMEOUT )
mock . assert_any_call ( url , * * kwargs )
2023-01-29 22:13:58 +00:00
2023-03-21 02:17:55 +00:00
def assert_object ( self , id , delivered_protocol = None , * * props ) :
2023-07-07 04:16:04 +00:00
ignore = props . pop ( ' ignore ' , [ ] )
2023-01-29 22:13:58 +00:00
got = Object . get_by_id ( id )
assert got , id
2023-02-01 21:19:41 +00:00
for field in ' delivered ' , ' undelivered ' , ' failed ' :
2023-03-21 02:17:55 +00:00
props [ field ] = [ Target ( uri = uri , protocol = delivered_protocol )
2023-02-01 21:19:41 +00:00
for uri in props . get ( field , [ ] ) ]
2023-07-07 04:16:04 +00:00
if ' our_as1 ' in props :
assert ' as2 ' not in props
assert ' bsky ' not in props
assert ' mf2 ' not in props
ignore . extend ( [ ' as2 ' , ' bsky ' , ' mf2 ' ] )
2023-02-24 03:17:26 +00:00
mf2 = props . get ( ' mf2 ' )
if mf2 and ' items ' in mf2 :
props [ ' mf2 ' ] = mf2 [ ' items ' ] [ 0 ]
2023-04-24 04:13:11 +00:00
type = props . pop ( ' type ' , None )
if type is not None :
self . assertEqual ( type , got . type )
object_ids = props . pop ( ' object_ids ' , None )
if object_ids is not None :
self . assertSetEqual ( set ( object_ids ) , set ( got . object_ids ) )
2023-02-14 05:43:49 +00:00
2023-02-24 03:17:26 +00:00
if expected_as1 := props . pop ( ' as1 ' , None ) :
self . assert_equals ( common . redirect_unwrap ( expected_as1 ) , got . as1 )
2023-04-02 02:13:51 +00:00
if got . mf2 :
got . mf2 . pop ( ' url ' , None )
2023-06-27 18:39:57 +00:00
for target in got . delivered :
del target . key
2023-01-29 22:13:58 +00:00
self . assert_entities_equal ( Object ( id = id , * * props ) , got ,
2023-07-16 21:06:03 +00:00
ignore = [ ' as1 ' , ' created ' , ' expire ' , ' labels ' ,
2023-06-21 03:59:32 +00:00
' object_ids ' , ' type ' , ' updated '
] + ignore )
2023-06-06 21:50:20 +00:00
return got
2023-04-02 02:13:51 +00:00
2023-05-31 20:17:17 +00:00
def assert_user ( self , cls , id , * * props ) :
got = cls . get_by_id ( id )
assert got , id
2023-06-16 04:22:20 +00:00
obj_as2 = props . pop ( ' obj_as2 ' , None )
if obj_as2 :
self . assert_equals ( obj_as2 , got . as2 ( ) )
# generated, computed, etc
2023-06-16 20:27:04 +00:00
ignore = [ ' created ' , ' mod ' , ' obj_key ' , ' p256_key ' , ' private_exponent ' ,
' public_exponent ' , ' readable_id ' , ' updated ' ]
2023-06-16 04:22:20 +00:00
for prop in ignore :
assert prop not in props
self . assert_entities_equal ( cls ( id = id , * * props ) , got , ignore = ignore )
2023-05-31 20:17:17 +00:00
if cls != ActivityPub :
assert got . mod
assert got . private_exponent
assert got . public_exponent
# if cls != ATProto:
# assert got.p256_key
2023-06-06 21:50:20 +00:00
return got
2023-04-02 02:13:51 +00:00
def assert_equals ( self , expected , actual , msg = None , ignore = ( ) , * * kwargs ) :
return super ( ) . assert_equals (
expected , actual , msg = msg , ignore = tuple ( ignore ) + ( ' @context ' , ) , * * kwargs )