From 4a0d413c006e7282f79db65ff1f2b748706a023d Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Thu, 19 Jan 2023 13:56:56 +0000 Subject: [PATCH] MOW-87: implement remote followers/following --- backend/src/activitypub/actors/follow.ts | 45 +++-- backend/src/activitypub/actors/outbox.ts | 32 +--- backend/src/activitypub/core.ts | 15 -- backend/src/activitypub/objects/collection.ts | 41 +++++ backend/src/mastodon/account.ts | 10 +- backend/test/mastodon/accounts.spec.ts | 161 ++++++++++++++++-- functions/api/v1/accounts/[id]/followers.ts | 66 +++++-- functions/api/v1/accounts/[id]/following.ts | 64 +++++-- 8 files changed, 330 insertions(+), 104 deletions(-) delete mode 100644 backend/src/activitypub/core.ts create mode 100644 backend/src/activitypub/objects/collection.ts diff --git a/backend/src/activitypub/actors/follow.ts b/backend/src/activitypub/actors/follow.ts index 141622d..29ab0c3 100644 --- a/backend/src/activitypub/actors/follow.ts +++ b/backend/src/activitypub/actors/follow.ts @@ -1,24 +1,35 @@ import type { Actor } from 'wildebeest/backend/src/activitypub/actors' -import type { OrderedCollection } from 'wildebeest/backend/src/activitypub/core' +import * as actors from 'wildebeest/backend/src/activitypub/actors' +import type { OrderedCollection } from 'wildebeest/backend/src/activitypub/objects/collection' +import { getMetadata, loadItems } from 'wildebeest/backend/src/activitypub/objects/collection' -const headers = { - accept: 'application/activity+json', +export async function countFollowing(actor: Actor): Promise { + const collection = await getMetadata(actor.following) + return collection.totalItems } -export async function getFollowingMetadata(actor: Actor): Promise> { - const res = await fetch(actor.following, { headers }) - if (!res.ok) { - throw new Error(`${actor.following} returned ${res.status}`) - } - - return res.json>() +export async function countFollowers(actor: Actor): Promise { + const collection = await getMetadata(actor.followers) + return collection.totalItems } -export async function getFollowersMetadata(actor: Actor): Promise> { - const res = await fetch(actor.followers, { headers }) - if (!res.ok) { - throw new Error(`${actor.followers} returned ${res.status}`) - } - - return res.json>() +export async function getFollowers(actor: Actor): Promise> { + const collection = await getMetadata(actor.followers) + collection.items = await loadItems(collection) + return collection +} + +export async function getFollowing(actor: Actor): Promise> { + const collection = await getMetadata(actor.following) + collection.items = await loadItems(collection) + return collection +} + +export async function loadActors(db: D1Database, collection: OrderedCollection): Promise> { + const promises = collection.items.map((item) => { + const actorId = new URL(item) + return actors.getAndCache(actorId, db) + }) + + return Promise.all(promises) } diff --git a/backend/src/activitypub/actors/outbox.ts b/backend/src/activitypub/actors/outbox.ts index ff67e0c..1deaf61 100644 --- a/backend/src/activitypub/actors/outbox.ts +++ b/backend/src/activitypub/actors/outbox.ts @@ -1,7 +1,8 @@ import type { Object } from 'wildebeest/backend/src/activitypub/objects' import type { Activity } from 'wildebeest/backend/src/activitypub/activities' import type { Actor } from 'wildebeest/backend/src/activitypub/actors' -import type { OrderedCollection, OrderedCollectionPage } from 'wildebeest/backend/src/activitypub/core' +import type { OrderedCollection } from 'wildebeest/backend/src/activitypub/objects/collection' +import { getMetadata, loadItems } from 'wildebeest/backend/src/activitypub/objects/collection' import { PUBLIC_GROUP } from 'wildebeest/backend/src/activitypub/activities' export async function addObjectInOutbox( @@ -30,35 +31,14 @@ export async function addObjectInOutbox( } } -const headers = { - accept: 'application/activity+json', -} - -export async function getMetadata(actor: Actor): Promise> { - const res = await fetch(actor.outbox, { headers }) - if (!res.ok) { - throw new Error(`${actor.outbox} returned ${res.status}`) - } - - return res.json>() -} - export async function get(actor: Actor): Promise> { - const collection = await getMetadata(actor) + const collection = await getMetadata(actor.outbox) collection.items = await loadItems(collection, 20) return collection } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -async function loadItems(collection: OrderedCollection, max: number): Promise> { - // FIXME: implement max and multi page support - - const res = await fetch(collection.first, { headers }) - if (!res.ok) { - throw new Error(`${collection.first} returned ${res.status}`) - } - - const data = await res.json>() - return data.orderedItems +export async function countStatuses(actor: Actor): Promise { + const metadata = await getMetadata(actor.outbox) + return metadata.totalItems } diff --git a/backend/src/activitypub/core.ts b/backend/src/activitypub/core.ts deleted file mode 100644 index 4fdb023..0000000 --- a/backend/src/activitypub/core.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { Object } from 'wildebeest/backend/src/activitypub/objects' - -export interface Collection extends Object { - totalItems: number - current?: string - first: URL - last: URL - items: Array -} - -export interface OrderedCollection extends Collection {} - -export interface OrderedCollectionPage extends Object { - orderedItems: Array -} diff --git a/backend/src/activitypub/objects/collection.ts b/backend/src/activitypub/objects/collection.ts new file mode 100644 index 0000000..daccbf7 --- /dev/null +++ b/backend/src/activitypub/objects/collection.ts @@ -0,0 +1,41 @@ +import type { Object } from 'wildebeest/backend/src/activitypub/objects' + +export interface Collection extends Object { + totalItems: number + current?: string + first: URL + last: URL + items: Array +} + +export interface OrderedCollection extends Collection {} + +export interface OrderedCollectionPage extends Object { + orderedItems: Array +} + +const headers = { + accept: 'application/activity+json', +} + +export async function getMetadata(url: URL): Promise> { + const res = await fetch(url, { headers }) + if (!res.ok) { + throw new Error(`${url} returned ${res.status}`) + } + + return res.json>() +} + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +export async function loadItems(collection: OrderedCollection, max?: number): Promise> { + // FIXME: implement max and multi page support + + const res = await fetch(collection.first, { headers }) + if (!res.ok) { + throw new Error(`${collection.first} returned ${res.status}`) + } + + const data = await res.json>() + return data.orderedItems +} diff --git a/backend/src/mastodon/account.ts b/backend/src/mastodon/account.ts index 7d2624a..2f6751c 100644 --- a/backend/src/mastodon/account.ts +++ b/backend/src/mastodon/account.ts @@ -42,14 +42,14 @@ function toMastodonAccount(acct: string, res: Actor): MastodonAccount { // Load an external user, using ActivityPub queries, and return it as a MastodonAccount export async function loadExternalMastodonAccount( acct: string, - res: Actor, + actor: Actor, loadStats: boolean = false ): Promise { - const account = toMastodonAccount(acct, res) + const account = toMastodonAccount(acct, actor) if (loadStats === true) { - account.statuses_count = (await apOutbox.getMetadata(res)).totalItems - account.followers_count = (await apFollow.getFollowersMetadata(res)).totalItems - account.following_count = (await apFollow.getFollowingMetadata(res)).totalItems + account.statuses_count = await apOutbox.countStatuses(actor) + account.followers_count = await apFollow.countFollowers(actor) + account.following_count = await apFollow.countFollowing(actor) } return account } diff --git a/backend/test/mastodon/accounts.spec.ts b/backend/test/mastodon/accounts.spec.ts index 2fea04f..b8bfbff 100644 --- a/backend/test/mastodon/accounts.spec.ts +++ b/backend/test/mastodon/accounts.spec.ts @@ -638,10 +638,82 @@ describe('Mastodon APIs', () => { test('get remote actor followers', async () => { const db = await makeDB() - const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') + const actorA = await createPerson(domain, db, userKEK, 'a@cloudflare.com') + + globalThis.fetch = async (input: any) => { + if (input.toString() === 'https://example.com/.well-known/webfinger?resource=acct%3Asven%40example.com') { + return new Response( + JSON.stringify({ + links: [ + { + rel: 'self', + type: 'application/activity+json', + href: 'https://example.com/users/sven', + }, + ], + }) + ) + } + + if (input.toString() === 'https://example.com/users/sven') { + return new Response( + JSON.stringify({ + id: 'https://example.com/users/sven', + type: 'Person', + followers: 'https://example.com/users/sven/followers', + }) + ) + } + + if (input.toString() === 'https://example.com/users/sven/followers') { + return new Response( + JSON.stringify({ + '@context': 'https://www.w3.org/ns/activitystreams', + id: 'https://example.com/users/sven/followers', + type: 'OrderedCollection', + totalItems: 3, + first: 'https://example.com/users/sven/followers/1', + }) + ) + } + + if (input.toString() === 'https://example.com/users/sven/followers/1') { + return new Response( + JSON.stringify({ + '@context': 'https://www.w3.org/ns/activitystreams', + id: 'https://example.com/users/sven/followers/1', + type: 'OrderedCollectionPage', + totalItems: 3, + partOf: 'https://example.com/users/sven/followers', + orderedItems: [ + actorA.id.toString(), // local user + 'https://example.com/users/b', // remote user + ], + }) + ) + } + + if (input.toString() === 'https://example.com/users/b') { + return new Response( + JSON.stringify({ + id: 'https://example.com/users/b', + type: 'Person', + }) + ) + } + + throw new Error('unexpected request to ' + input) + } + const req = new Request(`https://${domain}`) - const res = await accounts_followers.handleRequest(req, db, 'sven@example.com', connectedActor) - assert.equal(res.status, 403) + const res = await accounts_followers.handleRequest(req, db, 'sven@example.com') + assert.equal(res.status, 200) + + const data = await res.json>() + assert.equal(data.length, 2) + + assert.equal(data[0].acct, 'a@cloudflare.com') + assert.equal(data[1].acct, 'b@example.com') }) test('get local actor followers', async () => { @@ -664,9 +736,8 @@ describe('Mastodon APIs', () => { await addFollowing(db, actor2, actor, 'sven@' + domain) await acceptFollowing(db, actor2, actor) - const connectedActor = actor const req = new Request(`https://${domain}`) - const res = await accounts_followers.handleRequest(req, db, 'sven', connectedActor) + const res = await accounts_followers.handleRequest(req, db, 'sven') assert.equal(res.status, 200) const data = await res.json>() @@ -693,9 +764,8 @@ describe('Mastodon APIs', () => { await addFollowing(db, actor, actor2, 'sven@' + domain) await acceptFollowing(db, actor, actor2) - const connectedActor = actor const req = new Request(`https://${domain}`) - const res = await accounts_following.handleRequest(req, db, 'sven', connectedActor) + const res = await accounts_following.handleRequest(req, db, 'sven') assert.equal(res.status, 200) const data = await res.json>() @@ -704,11 +774,82 @@ describe('Mastodon APIs', () => { test('get remote actor following', async () => { const db = await makeDB() + const actorA = await createPerson(domain, db, userKEK, 'a@cloudflare.com') + + globalThis.fetch = async (input: any) => { + if (input.toString() === 'https://example.com/.well-known/webfinger?resource=acct%3Asven%40example.com') { + return new Response( + JSON.stringify({ + links: [ + { + rel: 'self', + type: 'application/activity+json', + href: 'https://example.com/users/sven', + }, + ], + }) + ) + } + + if (input.toString() === 'https://example.com/users/sven') { + return new Response( + JSON.stringify({ + id: 'https://example.com/users/sven', + type: 'Person', + following: 'https://example.com/users/sven/following', + }) + ) + } + + if (input.toString() === 'https://example.com/users/sven/following') { + return new Response( + JSON.stringify({ + '@context': 'https://www.w3.org/ns/activitystreams', + id: 'https://example.com/users/sven/following', + type: 'OrderedCollection', + totalItems: 3, + first: 'https://example.com/users/sven/following/1', + }) + ) + } + + if (input.toString() === 'https://example.com/users/sven/following/1') { + return new Response( + JSON.stringify({ + '@context': 'https://www.w3.org/ns/activitystreams', + id: 'https://example.com/users/sven/following/1', + type: 'OrderedCollectionPage', + totalItems: 3, + partOf: 'https://example.com/users/sven/following', + orderedItems: [ + actorA.id.toString(), // local user + 'https://example.com/users/b', // remote user + ], + }) + ) + } + + if (input.toString() === 'https://example.com/users/b') { + return new Response( + JSON.stringify({ + id: 'https://example.com/users/b', + type: 'Person', + }) + ) + } + + throw new Error('unexpected request to ' + input) + } - const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') const req = new Request(`https://${domain}`) - const res = await accounts_following.handleRequest(req, db, 'sven@example.com', connectedActor) - assert.equal(res.status, 403) + const res = await accounts_following.handleRequest(req, db, 'sven@example.com') + assert.equal(res.status, 200) + + const data = await res.json>() + assert.equal(data.length, 2) + + assert.equal(data[0].acct, 'a@cloudflare.com') + assert.equal(data[1].acct, 'b@example.com') }) test('get remote actor featured_tags', async () => { diff --git a/functions/api/v1/accounts/[id]/followers.ts b/functions/api/v1/accounts/[id]/followers.ts index da144b8..2887a4e 100644 --- a/functions/api/v1/accounts/[id]/followers.ts +++ b/functions/api/v1/accounts/[id]/followers.ts @@ -1,36 +1,70 @@ // https://docs.joinmastodon.org/methods/accounts/#followers +import type { Handle } from 'wildebeest/backend/src/utils/parse' +import { actorURL } from 'wildebeest/backend/src/activitypub/actors' import { cors } from 'wildebeest/backend/src/utils/cors' import { loadExternalMastodonAccount } from 'wildebeest/backend/src/mastodon/account' -import { urlToHandle } from 'wildebeest/backend/src/utils/handle' import { parseHandle } from 'wildebeest/backend/src/utils/parse' -import * as actors from 'wildebeest/backend/src/activitypub/actors' +import { urlToHandle } from 'wildebeest/backend/src/utils/handle' import { MastodonAccount } from 'wildebeest/backend/src/types/account' -import type { Person } from 'wildebeest/backend/src/activitypub/actors' import type { ContextData } from 'wildebeest/backend/src/types/context' -import { getFollowers } from 'wildebeest/backend/src/mastodon/follow' import type { Env } from 'wildebeest/backend/src/types/env' -import { domainNotAuthorized } from 'wildebeest/backend/src/errors' +import * as actors from 'wildebeest/backend/src/activitypub/actors' +import * as webfinger from 'wildebeest/backend/src/webfinger' +import { getFollowers, loadActors } from 'wildebeest/backend/src/activitypub/actors/follow' +import * as localFollow from 'wildebeest/backend/src/mastodon/follow' -export const onRequest: PagesFunction = async ({ params, request, env, data }) => { - return handleRequest(request, env.DATABASE, params.id as string, data.connectedActor) +export const onRequest: PagesFunction = async ({ params, request, env }) => { + return handleRequest(request, env.DATABASE, params.id as string) } -export async function handleRequest( - request: Request, - db: D1Database, - id: string, - connectedActor: Person -): Promise { +export async function handleRequest(request: Request, db: D1Database, id: string): Promise { const handle = parseHandle(id) const domain = new URL(request.url).hostname - if (handle.domain !== null && handle.domain !== domain) { - return domainNotAuthorized() + + if (handle.domain === null || (handle.domain !== null && handle.domain === domain)) { + // Retrieve the infos from a local user + return getLocalFollowers(request, handle, db) + } else if (handle.domain !== null) { + // Retrieve the infos of a remote actor + return getRemoteFollowers(request, handle, db) + } else { + return new Response('', { status: 403 }) + } +} + +async function getRemoteFollowers(request: Request, handle: Handle, db: D1Database): Promise { + const acct = `${handle.localPart}@${handle.domain}` + const link = await webfinger.queryAcctLink(handle.domain!, acct) + if (link === null) { + return new Response('', { status: 404 }) } + const actor = await actors.getAndCache(link, db) + const followersIds = await getFollowers(actor) + const followers = await loadActors(db, followersIds) + + const promises = followers.map((actor) => { + const acct = urlToHandle(actor.id) + return loadExternalMastodonAccount(acct, actor, false) + }) + + const out = await Promise.all(promises) + const headers = { + ...cors(), + 'content-type': 'application/json; charset=utf-8', + } + return new Response(JSON.stringify(out), { headers }) +} + +async function getLocalFollowers(request: Request, handle: Handle, db: D1Database): Promise { + const domain = new URL(request.url).hostname + const actorId = actorURL(domain, handle.localPart) + const actor = await actors.getAndCache(actorId, db) + + const followers = await localFollow.getFollowers(db, actor) const out: Array = [] - const followers = await getFollowers(db, connectedActor) for (let i = 0, len = followers.length; i < len; i++) { const id = new URL(followers[i]) diff --git a/functions/api/v1/accounts/[id]/following.ts b/functions/api/v1/accounts/[id]/following.ts index 23c6f41..ae53f85 100644 --- a/functions/api/v1/accounts/[id]/following.ts +++ b/functions/api/v1/accounts/[id]/following.ts @@ -1,36 +1,70 @@ // https://docs.joinmastodon.org/methods/accounts/#following +import type { Handle } from 'wildebeest/backend/src/utils/parse' +import { actorURL } from 'wildebeest/backend/src/activitypub/actors' import { cors } from 'wildebeest/backend/src/utils/cors' import { loadExternalMastodonAccount } from 'wildebeest/backend/src/mastodon/account' import { parseHandle } from 'wildebeest/backend/src/utils/parse' import { urlToHandle } from 'wildebeest/backend/src/utils/handle' -import * as actors from 'wildebeest/backend/src/activitypub/actors' import { MastodonAccount } from 'wildebeest/backend/src/types/account' -import type { Person } from 'wildebeest/backend/src/activitypub/actors' import type { ContextData } from 'wildebeest/backend/src/types/context' -import { getFollowingId } from 'wildebeest/backend/src/mastodon/follow' +import * as localFollow from 'wildebeest/backend/src/mastodon/follow' import type { Env } from 'wildebeest/backend/src/types/env' -import { domainNotAuthorized } from 'wildebeest/backend/src/errors' +import * as actors from 'wildebeest/backend/src/activitypub/actors' +import * as webfinger from 'wildebeest/backend/src/webfinger' +import { getFollowing, loadActors } from 'wildebeest/backend/src/activitypub/actors/follow' -export const onRequest: PagesFunction = async ({ params, request, env, data }) => { - return handleRequest(request, env.DATABASE, params.id as string, data.connectedActor) +export const onRequest: PagesFunction = async ({ params, request, env }) => { + return handleRequest(request, env.DATABASE, params.id as string) } -export async function handleRequest( - request: Request, - db: D1Database, - id: string, - connectedActor: Person -): Promise { +export async function handleRequest(request: Request, db: D1Database, id: string): Promise { const handle = parseHandle(id) const domain = new URL(request.url).hostname - if (handle.domain !== null && handle.domain !== domain) { - return domainNotAuthorized() + + if (handle.domain === null || (handle.domain !== null && handle.domain === domain)) { + // Retrieve the infos from a local user + return getLocalFollowing(request, handle, db) + } else if (handle.domain !== null) { + // Retrieve the infos of a remote actor + return getRemoteFollowing(request, handle, db) + } else { + return new Response('', { status: 403 }) + } +} + +async function getRemoteFollowing(request: Request, handle: Handle, db: D1Database): Promise { + const acct = `${handle.localPart}@${handle.domain}` + const link = await webfinger.queryAcctLink(handle.domain!, acct) + if (link === null) { + return new Response('', { status: 404 }) } + const actor = await actors.getAndCache(link, db) + const followingIds = await getFollowing(actor) + const following = await loadActors(db, followingIds) + + const promises = following.map((actor) => { + const acct = urlToHandle(actor.id) + return loadExternalMastodonAccount(acct, actor, false) + }) + + const out = await Promise.all(promises) + const headers = { + ...cors(), + 'content-type': 'application/json; charset=utf-8', + } + return new Response(JSON.stringify(out), { headers }) +} + +async function getLocalFollowing(request: Request, handle: Handle, db: D1Database): Promise { + const domain = new URL(request.url).hostname + const actorId = actorURL(domain, handle.localPart) + const actor = await actors.getAndCache(actorId, db) + + const following = await localFollow.getFollowingId(db, actor) const out: Array = [] - const following = await getFollowingId(db, connectedActor) for (let i = 0, len = following.length; i < len; i++) { const id = new URL(following[i])