From d89ed5a8af2629feadac93c6b954717f331142bb Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Mon, 27 Feb 2023 17:37:03 +0000 Subject: [PATCH 01/62] make getDatabase async --- backend/src/database/index.ts | 2 +- backend/src/middleware/main.ts | 2 +- consumer/src/deliver.ts | 4 ++-- consumer/src/inbox.ts | 2 +- consumer/src/index.ts | 2 +- frontend/src/routes/(admin)/oauth/authorize/index.tsx | 6 +++--- .../src/routes/(frontend)/[accountId]/[statusId]/index.tsx | 4 ++-- frontend/src/routes/(frontend)/[accountId]/index.tsx | 2 +- frontend/src/routes/(frontend)/[accountId]/layout.tsx | 4 ++-- .../routes/(frontend)/[accountId]/with_replies/index.tsx | 2 +- frontend/src/routes/(frontend)/explore/index.tsx | 2 +- frontend/src/routes/(frontend)/public/index.tsx | 2 +- frontend/src/routes/(frontend)/public/local/index.tsx | 2 +- functions/.well-known/webfinger.ts | 2 +- functions/ap/o/[id].ts | 2 +- functions/ap/users/[id].ts | 2 +- functions/ap/users/[id]/followers.ts | 2 +- functions/ap/users/[id]/followers/page.ts | 2 +- functions/ap/users/[id]/following.ts | 2 +- functions/ap/users/[id]/following/page.ts | 2 +- functions/ap/users/[id]/inbox.ts | 2 +- functions/ap/users/[id]/outbox.ts | 2 +- functions/ap/users/[id]/outbox/page.ts | 2 +- functions/api/v1/accounts/[id].ts | 2 +- functions/api/v1/accounts/[id]/follow.ts | 2 +- functions/api/v1/accounts/[id]/followers.ts | 2 +- functions/api/v1/accounts/[id]/following.ts | 2 +- functions/api/v1/accounts/[id]/statuses.ts | 2 +- functions/api/v1/accounts/[id]/unfollow.ts | 2 +- functions/api/v1/accounts/relationships.ts | 2 +- functions/api/v1/accounts/update_credentials.ts | 2 +- functions/api/v1/accounts/verify_credentials.ts | 2 +- functions/api/v1/apps.ts | 2 +- functions/api/v1/instance/peers.ts | 2 +- functions/api/v1/notifications/[id].ts | 2 +- functions/api/v1/push/subscription.ts | 4 ++-- functions/api/v1/statuses.ts | 2 +- functions/api/v1/statuses/[id].ts | 4 ++-- functions/api/v1/statuses/[id]/context.ts | 2 +- functions/api/v1/statuses/[id]/favourite.ts | 2 +- functions/api/v1/statuses/[id]/reblog.ts | 2 +- functions/api/v1/tags/[tag].ts | 2 +- functions/api/v1/timelines/public.ts | 2 +- functions/api/v1/timelines/tag/[tag].ts | 2 +- functions/api/v2/instance.ts | 2 +- functions/api/v2/media.ts | 2 +- functions/api/v2/media/[id].ts | 2 +- functions/api/v2/search.ts | 2 +- functions/api/wb/settings/account/alias.ts | 2 +- functions/first-login.ts | 2 +- functions/oauth/authorize.ts | 2 +- functions/oauth/token.ts | 2 +- 52 files changed, 59 insertions(+), 59 deletions(-) diff --git a/backend/src/database/index.ts b/backend/src/database/index.ts index 260409e..01ab295 100644 --- a/backend/src/database/index.ts +++ b/backend/src/database/index.ts @@ -23,6 +23,6 @@ export interface PreparedStatement { raw(): Promise } -export function getDatabase(env: Pick): Database { +export async function getDatabase(env: Pick): Promise { return d1(env) } diff --git a/backend/src/middleware/main.ts b/backend/src/middleware/main.ts index dd10e80..f226e8b 100644 --- a/backend/src/middleware/main.ts +++ b/backend/src/middleware/main.ts @@ -97,7 +97,7 @@ export async function main(context: EventContext) { // configuration, which are used to verify the JWT. // TODO: since we don't load the instance configuration anymore, we // don't need to load the user before anymore. - if (!(await loadContextData(getDatabase(context.env), clientId, payload.email, context))) { + if (!(await loadContextData(await getDatabase(context.env), clientId, payload.email, context))) { return errors.notAuthorized('failed to load context data') } diff --git a/consumer/src/deliver.ts b/consumer/src/deliver.ts index c033825..ca0ed22 100644 --- a/consumer/src/deliver.ts +++ b/consumer/src/deliver.ts @@ -8,12 +8,12 @@ import { deliverToActor } from 'wildebeest/backend/src/activitypub/deliver' export async function handleDeliverMessage(env: Env, actor: Actor, message: DeliverMessageBody) { const toActorId = new URL(message.toActorId) - const targetActor = await actors.getAndCache(toActorId, getDatabase(env)) + const targetActor = await actors.getAndCache(toActorId, await getDatabase(env)) if (targetActor === null) { console.warn(`actor ${toActorId} not found`) return } - const signingKey = await getSigningKey(message.userKEK, getDatabase(env), actor) + const signingKey = await getSigningKey(message.userKEK, await getDatabase(env), actor) await deliverToActor(signingKey, actor, targetActor, message.activity, env.DOMAIN) } diff --git a/consumer/src/inbox.ts b/consumer/src/inbox.ts index 35687c4..5ef3227 100644 --- a/consumer/src/inbox.ts +++ b/consumer/src/inbox.ts @@ -9,7 +9,7 @@ import type { Env } from './' export async function handleInboxMessage(env: Env, actor: Actor, message: InboxMessageBody) { const domain = env.DOMAIN - const db = getDatabase(env) + const db = await getDatabase(env) const adminEmail = env.ADMIN_EMAIL const cache = cacheFromEnv(env) const activity = message.activity diff --git a/consumer/src/index.ts b/consumer/src/index.ts index bf070b1..fa6d72e 100644 --- a/consumer/src/index.ts +++ b/consumer/src/index.ts @@ -20,7 +20,7 @@ export type Env = { export default { async queue(batch: MessageBatch, env: Env, ctx: ExecutionContext) { const sentry = initSentryQueue(env, ctx) - const db = getDatabase(env) + const db = await getDatabase(env) try { for (const message of batch.messages) { diff --git a/frontend/src/routes/(admin)/oauth/authorize/index.tsx b/frontend/src/routes/(admin)/oauth/authorize/index.tsx index 0085774..e2fc9fb 100644 --- a/frontend/src/routes/(admin)/oauth/authorize/index.tsx +++ b/frontend/src/routes/(admin)/oauth/authorize/index.tsx @@ -14,7 +14,7 @@ export const clientLoader = loader$, { DATABASE: D1Database }>(a const client_id = query.get('client_id') || '' let client: Client | null = null try { - client = await getClientById(getDatabase(platform), client_id) + client = await getClientById(await getDatabase(platform), client_id) } catch (e: unknown) { const error = e as { stack: string; cause: string } console.warn(error.stack, error.cause) @@ -49,10 +49,10 @@ export const userLoader = loader$< throw html(500, getErrorHtml("The Access JWT doesn't contain an email")) } - const person = await getPersonByEmail(getDatabase(platform), payload.email) + const person = await getPersonByEmail(await getDatabase(platform), payload.email) if (person === null) { const isFirstLogin = true - const res = await buildRedirect(getDatabase(platform), request as Request, isFirstLogin, jwt.value) + const res = await buildRedirect(await getDatabase(platform), request as Request, isFirstLogin, jwt.value) if (res.status === 302) { throw redirect(302, res.headers.get('location') || '') } else { diff --git a/frontend/src/routes/(frontend)/[accountId]/[statusId]/index.tsx b/frontend/src/routes/(frontend)/[accountId]/[statusId]/index.tsx index 06951cd..0b9a3b2 100644 --- a/frontend/src/routes/(frontend)/[accountId]/[statusId]/index.tsx +++ b/frontend/src/routes/(frontend)/[accountId]/[statusId]/index.tsx @@ -19,7 +19,7 @@ export const statusLoader = loader$< let statusText = '' try { const statusResponse = await statusAPI.handleRequestGet( - getDatabase(platform), + await getDatabase(platform), params.statusId, domain, {} as Person @@ -37,7 +37,7 @@ export const statusLoader = loader$< const statusTextContent = await getTextContent(status.content) try { - const contextResponse = await contextAPI.handleRequest(domain, getDatabase(platform), params.statusId) + const contextResponse = await contextAPI.handleRequest(domain, await getDatabase(platform), params.statusId) const contextText = await contextResponse.text() const context = JSON.parse(contextText ?? null) as StatusContext | null if (!context) { diff --git a/frontend/src/routes/(frontend)/[accountId]/index.tsx b/frontend/src/routes/(frontend)/[accountId]/index.tsx index 8cf8de8..7afb0d2 100644 --- a/frontend/src/routes/(frontend)/[accountId]/index.tsx +++ b/frontend/src/routes/(frontend)/[accountId]/index.tsx @@ -22,7 +22,7 @@ export const statusesLoader = loader$< const handle = parseHandle(accountId) accountId = handle.localPart - const response = await getLocalStatuses(request as Request, getDatabase(platform), handle, 0, false) + const response = await getLocalStatuses(request as Request, await getDatabase(platform), handle, 0, false) statuses = await response.json>() } catch { throw html( diff --git a/frontend/src/routes/(frontend)/[accountId]/layout.tsx b/frontend/src/routes/(frontend)/[accountId]/layout.tsx index 7985839..109b766 100644 --- a/frontend/src/routes/(frontend)/[accountId]/layout.tsx +++ b/frontend/src/routes/(frontend)/[accountId]/layout.tsx @@ -25,14 +25,14 @@ export const accountPageLoader = loader$< const accountId = url.pathname.split('/')[1] try { - const statusResponse = await statusAPI.handleRequestGet(getDatabase(platform), params.statusId, domain) + const statusResponse = await statusAPI.handleRequestGet(await getDatabase(platform), params.statusId, domain) const statusText = await statusResponse.text() isValidStatus = !!statusText } catch { isValidStatus = false } - account = await getAccount(domain, accountId, getDatabase(platform)) + account = await getAccount(domain, accountId, await getDatabase(platform)) } catch { throw html( 500, diff --git a/frontend/src/routes/(frontend)/[accountId]/with_replies/index.tsx b/frontend/src/routes/(frontend)/[accountId]/with_replies/index.tsx index 8009950..7d89c9a 100644 --- a/frontend/src/routes/(frontend)/[accountId]/with_replies/index.tsx +++ b/frontend/src/routes/(frontend)/[accountId]/with_replies/index.tsx @@ -23,7 +23,7 @@ export const statusesLoader = loader$< const handle = parseHandle(accountId) accountId = handle.localPart - const response = await getLocalStatuses(request as Request, getDatabase(platform), handle, 0, true) + const response = await getLocalStatuses(request as Request, await getDatabase(platform), handle, 0, true) statuses = await response.json>() } catch { throw html( diff --git a/frontend/src/routes/(frontend)/explore/index.tsx b/frontend/src/routes/(frontend)/explore/index.tsx index 932f4ff..cd39c0a 100644 --- a/frontend/src/routes/(frontend)/explore/index.tsx +++ b/frontend/src/routes/(frontend)/explore/index.tsx @@ -11,7 +11,7 @@ export const statusesLoader = loader$, { DATABASE: D1D async ({ platform, html }) => { try { // TODO: use the "trending" API endpoint here. - const response = await timelines.handleRequest(platform.domain, getDatabase(platform)) + const response = await timelines.handleRequest(platform.domain, await getDatabase(platform)) const results = await response.text() // Manually parse the JSON to ensure that Qwik finds the resulting objects serializable. return JSON.parse(results) as MastodonStatus[] diff --git a/frontend/src/routes/(frontend)/public/index.tsx b/frontend/src/routes/(frontend)/public/index.tsx index f32a349..cbb33d7 100644 --- a/frontend/src/routes/(frontend)/public/index.tsx +++ b/frontend/src/routes/(frontend)/public/index.tsx @@ -12,7 +12,7 @@ export const statusesLoader = loader$, { DATABASE: D1D async ({ platform, html }) => { try { // TODO: use the "trending" API endpoint here. - const response = await timelines.handleRequest(platform.domain, getDatabase(platform)) + const response = await timelines.handleRequest(platform.domain, await getDatabase(platform)) const results = await response.text() // Manually parse the JSON to ensure that Qwik finds the resulting objects serializable. return JSON.parse(results) as MastodonStatus[] diff --git a/frontend/src/routes/(frontend)/public/local/index.tsx b/frontend/src/routes/(frontend)/public/local/index.tsx index f756133..c2e8833 100644 --- a/frontend/src/routes/(frontend)/public/local/index.tsx +++ b/frontend/src/routes/(frontend)/public/local/index.tsx @@ -12,7 +12,7 @@ export const statusesLoader = loader$, { DATABASE: D1D async ({ platform, html }) => { try { // TODO: use the "trending" API endpoint here. - const response = await timelines.handleRequest(platform.domain, getDatabase(platform), { local: true }) + const response = await timelines.handleRequest(platform.domain, await getDatabase(platform), { local: true }) const results = await response.text() // Manually parse the JSON to ensure that Qwik finds the resulting objects serializable. return JSON.parse(results) as MastodonStatus[] diff --git a/functions/.well-known/webfinger.ts b/functions/.well-known/webfinger.ts index 9786aba..72d79e1 100644 --- a/functions/.well-known/webfinger.ts +++ b/functions/.well-known/webfinger.ts @@ -7,7 +7,7 @@ import type { WebFingerResponse } from '../../backend/src/webfinger' import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequest: PagesFunction = async ({ request, env }) => { - return handleRequest(request, getDatabase(env)) + return handleRequest(request, await getDatabase(env)) } const headers = { diff --git a/functions/ap/o/[id].ts b/functions/ap/o/[id].ts index 6ead150..8c401ff 100644 --- a/functions/ap/o/[id].ts +++ b/functions/ap/o/[id].ts @@ -5,7 +5,7 @@ import * as objects from 'wildebeest/backend/src/activitypub/objects' export const onRequest: PagesFunction = async ({ params, request, env }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } const headers = { diff --git a/functions/ap/users/[id].ts b/functions/ap/users/[id].ts index 3ef7862..ddfba06 100644 --- a/functions/ap/users/[id].ts +++ b/functions/ap/users/[id].ts @@ -7,7 +7,7 @@ import * as actors from 'wildebeest/backend/src/activitypub/actors' export const onRequest: PagesFunction = async ({ params, request, env }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } const headers = { diff --git a/functions/ap/users/[id]/followers.ts b/functions/ap/users/[id]/followers.ts index 931be69..b865bad 100644 --- a/functions/ap/users/[id]/followers.ts +++ b/functions/ap/users/[id]/followers.ts @@ -11,7 +11,7 @@ const headers = { export const onRequest: PagesFunction = async ({ params, request, env }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } export async function handleRequest(domain: string, db: Database, id: string): Promise { diff --git a/functions/ap/users/[id]/followers/page.ts b/functions/ap/users/[id]/followers/page.ts index 499b1f3..ff23c70 100644 --- a/functions/ap/users/[id]/followers/page.ts +++ b/functions/ap/users/[id]/followers/page.ts @@ -8,7 +8,7 @@ import type { Env } from 'wildebeest/backend/src/types/env' export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } const headers = { diff --git a/functions/ap/users/[id]/following.ts b/functions/ap/users/[id]/following.ts index 5e40c08..0e09c44 100644 --- a/functions/ap/users/[id]/following.ts +++ b/functions/ap/users/[id]/following.ts @@ -11,7 +11,7 @@ const headers = { export const onRequest: PagesFunction = async ({ params, request, env }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } export async function handleRequest(domain: string, db: Database, id: string): Promise { diff --git a/functions/ap/users/[id]/following/page.ts b/functions/ap/users/[id]/following/page.ts index f79ee9c..bef7e30 100644 --- a/functions/ap/users/[id]/following/page.ts +++ b/functions/ap/users/[id]/following/page.ts @@ -8,7 +8,7 @@ import type { Env } from 'wildebeest/backend/src/types/env' export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } const headers = { diff --git a/functions/ap/users/[id]/inbox.ts b/functions/ap/users/[id]/inbox.ts index 0913eae..52bd370 100644 --- a/functions/ap/users/[id]/inbox.ts +++ b/functions/ap/users/[id]/inbox.ts @@ -41,7 +41,7 @@ export const onRequest: PagesFunction = async ({ params, request, env const domain = new URL(request.url).hostname return handleRequest( domain, - getDatabase(env), + await getDatabase(env), params.id as string, activity, env.QUEUE, diff --git a/functions/ap/users/[id]/outbox.ts b/functions/ap/users/[id]/outbox.ts index 8ba8d9a..a7717e4 100644 --- a/functions/ap/users/[id]/outbox.ts +++ b/functions/ap/users/[id]/outbox.ts @@ -7,7 +7,7 @@ import type { Env } from 'wildebeest/backend/src/types/env' export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string, env.userKEK) + return handleRequest(domain, await getDatabase(env), params.id as string, env.userKEK) } const headers = { diff --git a/functions/ap/users/[id]/outbox/page.ts b/functions/ap/users/[id]/outbox/page.ts index 6b91c55..b574a63 100644 --- a/functions/ap/users/[id]/outbox/page.ts +++ b/functions/ap/users/[id]/outbox/page.ts @@ -12,7 +12,7 @@ import { PUBLIC_GROUP } from 'wildebeest/backend/src/activitypub/activities' export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } const headers = { diff --git a/functions/api/v1/accounts/[id].ts b/functions/api/v1/accounts/[id].ts index e3861d1..8101380 100644 --- a/functions/api/v1/accounts/[id].ts +++ b/functions/api/v1/accounts/[id].ts @@ -13,7 +13,7 @@ const headers = { export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, params.id as string, getDatabase(env)) + return handleRequest(domain, params.id as string, await getDatabase(env)) } export async function handleRequest(domain: string, id: string, db: Database): Promise { diff --git a/functions/api/v1/accounts/[id]/follow.ts b/functions/api/v1/accounts/[id]/follow.ts index ad74505..da9b916 100644 --- a/functions/api/v1/accounts/[id]/follow.ts +++ b/functions/api/v1/accounts/[id]/follow.ts @@ -13,7 +13,7 @@ import type { Relationship } from 'wildebeest/backend/src/types/account' import { addFollowing } from 'wildebeest/backend/src/mastodon/follow' export const onRequest: PagesFunction = async ({ request, env, params, data }) => { - return handleRequest(request, getDatabase(env), params.id as string, data.connectedActor, env.userKEK) + return handleRequest(request, await getDatabase(env), params.id as string, data.connectedActor, env.userKEK) } export async function handleRequest( diff --git a/functions/api/v1/accounts/[id]/followers.ts b/functions/api/v1/accounts/[id]/followers.ts index 21b4e6d..f093537 100644 --- a/functions/api/v1/accounts/[id]/followers.ts +++ b/functions/api/v1/accounts/[id]/followers.ts @@ -16,7 +16,7 @@ import { getFollowers, loadActors } from 'wildebeest/backend/src/activitypub/act import * as localFollow from 'wildebeest/backend/src/mastodon/follow' export const onRequest: PagesFunction = async ({ params, request, env }) => { - return handleRequest(request, getDatabase(env), params.id as string) + return handleRequest(request, await getDatabase(env), params.id as string) } export async function handleRequest(request: Request, db: Database, id: string): Promise { diff --git a/functions/api/v1/accounts/[id]/following.ts b/functions/api/v1/accounts/[id]/following.ts index 8db6e49..cc2ecac 100644 --- a/functions/api/v1/accounts/[id]/following.ts +++ b/functions/api/v1/accounts/[id]/following.ts @@ -16,7 +16,7 @@ import * as webfinger from 'wildebeest/backend/src/webfinger' import { getFollowing, loadActors } from 'wildebeest/backend/src/activitypub/actors/follow' export const onRequest: PagesFunction = async ({ params, request, env }) => { - return handleRequest(request, getDatabase(env), params.id as string) + return handleRequest(request, await getDatabase(env), params.id as string) } export async function handleRequest(request: Request, db: Database, id: string): Promise { diff --git a/functions/api/v1/accounts/[id]/statuses.ts b/functions/api/v1/accounts/[id]/statuses.ts index a79fe5e..ae44cfe 100644 --- a/functions/api/v1/accounts/[id]/statuses.ts +++ b/functions/api/v1/accounts/[id]/statuses.ts @@ -26,7 +26,7 @@ const headers = { } export const onRequest: PagesFunction = async ({ request, env, params }) => { - return handleRequest(request, getDatabase(env), params.id as string) + return handleRequest(request, await getDatabase(env), params.id as string) } export async function handleRequest(request: Request, db: Database, id: string): Promise { diff --git a/functions/api/v1/accounts/[id]/unfollow.ts b/functions/api/v1/accounts/[id]/unfollow.ts index 693022b..3b8ca1f 100644 --- a/functions/api/v1/accounts/[id]/unfollow.ts +++ b/functions/api/v1/accounts/[id]/unfollow.ts @@ -12,7 +12,7 @@ import type { Relationship } from 'wildebeest/backend/src/types/account' import { removeFollowing } from 'wildebeest/backend/src/mastodon/follow' export const onRequest: PagesFunction = async ({ request, env, params, data }) => { - return handleRequest(request, getDatabase(env), params.id as string, data.connectedActor, env.userKEK) + return handleRequest(request, await getDatabase(env), params.id as string, data.connectedActor, env.userKEK) } export async function handleRequest( diff --git a/functions/api/v1/accounts/relationships.ts b/functions/api/v1/accounts/relationships.ts index 0894f2d..1bbf96c 100644 --- a/functions/api/v1/accounts/relationships.ts +++ b/functions/api/v1/accounts/relationships.ts @@ -8,7 +8,7 @@ import type { ContextData } from 'wildebeest/backend/src/types/context' import { getFollowingAcct, getFollowingRequestedAcct } from 'wildebeest/backend/src/mastodon/follow' export const onRequest: PagesFunction = async ({ request, env, data }) => { - return handleRequest(request, getDatabase(env), data.connectedActor) + return handleRequest(request, await getDatabase(env), data.connectedActor) } export async function handleRequest(req: Request, db: Database, connectedActor: Person): Promise { diff --git a/functions/api/v1/accounts/update_credentials.ts b/functions/api/v1/accounts/update_credentials.ts index 82991c3..e015c97 100644 --- a/functions/api/v1/accounts/update_credentials.ts +++ b/functions/api/v1/accounts/update_credentials.ts @@ -22,7 +22,7 @@ const headers = { export const onRequest: PagesFunction = async ({ request, data, env }) => { return handleRequest( - getDatabase(env), + await getDatabase(env), request, data.connectedActor, env.CF_ACCOUNT_ID, diff --git a/functions/api/v1/accounts/verify_credentials.ts b/functions/api/v1/accounts/verify_credentials.ts index 0cd2ecb..84d06f7 100644 --- a/functions/api/v1/accounts/verify_credentials.ts +++ b/functions/api/v1/accounts/verify_credentials.ts @@ -12,7 +12,7 @@ export const onRequest: PagesFunction = async ({ data, en if (!data.connectedActor) { return errors.notAuthorized('no connected user') } - const user = await loadLocalMastodonAccount(getDatabase(env), data.connectedActor) + const user = await loadLocalMastodonAccount(await getDatabase(env), data.connectedActor) const res: CredentialAccount = { ...user, diff --git a/functions/api/v1/apps.ts b/functions/api/v1/apps.ts index c341820..32d9f17 100644 --- a/functions/api/v1/apps.ts +++ b/functions/api/v1/apps.ts @@ -17,7 +17,7 @@ type AppsPost = { } export const onRequest: PagesFunction = async ({ request, env }) => { - return handleRequest(getDatabase(env), request, getVAPIDKeys(env)) + return handleRequest(await getDatabase(env), request, getVAPIDKeys(env)) } export async function handleRequest(db: Database, request: Request, vapidKeys: JWK) { diff --git a/functions/api/v1/instance/peers.ts b/functions/api/v1/instance/peers.ts index e5535a6..d087d76 100644 --- a/functions/api/v1/instance/peers.ts +++ b/functions/api/v1/instance/peers.ts @@ -4,7 +4,7 @@ import type { Env } from 'wildebeest/backend/src/types/env' import { getPeers } from 'wildebeest/backend/src/activitypub/peers' export const onRequest: PagesFunction = async ({ env }) => { - return handleRequest(getDatabase(env)) + return handleRequest(await getDatabase(env)) } export async function handleRequest(db: Database): Promise { diff --git a/functions/api/v1/notifications/[id].ts b/functions/api/v1/notifications/[id].ts index 2a1fff4..af67df3 100644 --- a/functions/api/v1/notifications/[id].ts +++ b/functions/api/v1/notifications/[id].ts @@ -15,7 +15,7 @@ const headers = { export const onRequest: PagesFunction = async ({ data, request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, params.id as string, getDatabase(env), data.connectedActor) + return handleRequest(domain, params.id as string, await getDatabase(env), data.connectedActor) } export async function handleRequest( diff --git a/functions/api/v1/push/subscription.ts b/functions/api/v1/push/subscription.ts index ad83cb0..c5ba357 100644 --- a/functions/api/v1/push/subscription.ts +++ b/functions/api/v1/push/subscription.ts @@ -12,11 +12,11 @@ import { VAPIDPublicKey } from 'wildebeest/backend/src/mastodon/subscription' import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequestGet: PagesFunction = async ({ request, env, data }) => { - return handleGetRequest(getDatabase(env), request, data.connectedActor, data.clientId, getVAPIDKeys(env)) + return handleGetRequest(await getDatabase(env), request, data.connectedActor, data.clientId, getVAPIDKeys(env)) } export const onRequestPost: PagesFunction = async ({ request, env, data }) => { - return handlePostRequest(getDatabase(env), request, data.connectedActor, data.clientId, getVAPIDKeys(env)) + return handlePostRequest(await getDatabase(env), request, data.connectedActor, data.clientId, getVAPIDKeys(env)) } const headers = { diff --git a/functions/api/v1/statuses.ts b/functions/api/v1/statuses.ts index 3b7ba94..15c0207 100644 --- a/functions/api/v1/statuses.ts +++ b/functions/api/v1/statuses.ts @@ -39,7 +39,7 @@ type StatusCreate = { } export const onRequest: PagesFunction = async ({ request, env, data }) => { - return handleRequest(request, getDatabase(env), data.connectedActor, env.userKEK, env.QUEUE, cacheFromEnv(env)) + return handleRequest(request, await getDatabase(env), data.connectedActor, env.userKEK, env.QUEUE, cacheFromEnv(env)) } // FIXME: add tests for delivery to followers and mentions to a specific Actor. diff --git a/functions/api/v1/statuses/[id].ts b/functions/api/v1/statuses/[id].ts index c813bfa..7194978 100644 --- a/functions/api/v1/statuses/[id].ts +++ b/functions/api/v1/statuses/[id].ts @@ -20,13 +20,13 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequestGet: PagesFunction = async ({ params, env, request, data }) => { const domain = new URL(request.url).hostname - return handleRequestGet(getDatabase(env), params.id as UUID, domain, data.connectedActor) + return handleRequestGet(await getDatabase(env), params.id as UUID, domain, data.connectedActor) } export const onRequestDelete: PagesFunction = async ({ params, env, request, data }) => { const domain = new URL(request.url).hostname return handleRequestDelete( - getDatabase(env), + await getDatabase(env), params.id as UUID, data.connectedActor, domain, diff --git a/functions/api/v1/statuses/[id]/context.ts b/functions/api/v1/statuses/[id]/context.ts index 710fda6..e5af0df 100644 --- a/functions/api/v1/statuses/[id]/context.ts +++ b/functions/api/v1/statuses/[id]/context.ts @@ -10,7 +10,7 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), params.id as string) + return handleRequest(domain, await getDatabase(env), params.id as string) } const headers = { diff --git a/functions/api/v1/statuses/[id]/favourite.ts b/functions/api/v1/statuses/[id]/favourite.ts index 5ec5692..10c9080 100644 --- a/functions/api/v1/statuses/[id]/favourite.ts +++ b/functions/api/v1/statuses/[id]/favourite.ts @@ -17,7 +17,7 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequest: PagesFunction = async ({ env, data, params, request }) => { const domain = new URL(request.url).hostname - return handleRequest(getDatabase(env), params.id as string, data.connectedActor, env.userKEK, domain) + return handleRequest(await getDatabase(env), params.id as string, data.connectedActor, env.userKEK, domain) } export async function handleRequest( diff --git a/functions/api/v1/statuses/[id]/reblog.ts b/functions/api/v1/statuses/[id]/reblog.ts index 15296ce..456437b 100644 --- a/functions/api/v1/statuses/[id]/reblog.ts +++ b/functions/api/v1/statuses/[id]/reblog.ts @@ -17,7 +17,7 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequest: PagesFunction = async ({ env, data, params, request }) => { const domain = new URL(request.url).hostname - return handleRequest(getDatabase(env), params.id as string, data.connectedActor, env.userKEK, env.QUEUE, domain) + return handleRequest(await getDatabase(env), params.id as string, data.connectedActor, env.userKEK, env.QUEUE, domain) } export async function handleRequest( diff --git a/functions/api/v1/tags/[tag].ts b/functions/api/v1/tags/[tag].ts index 6c2b9eb..8b71c04 100644 --- a/functions/api/v1/tags/[tag].ts +++ b/functions/api/v1/tags/[tag].ts @@ -14,7 +14,7 @@ const headers = { export const onRequestGet: PagesFunction = async ({ params, env, request }) => { const domain = new URL(request.url).hostname - return handleRequestGet(getDatabase(env), domain, params.tag as string) + return handleRequestGet(await getDatabase(env), domain, params.tag as string) } export async function handleRequestGet(db: Database, domain: string, value: string): Promise { diff --git a/functions/api/v1/timelines/public.ts b/functions/api/v1/timelines/public.ts index ec946c1..2b2dc96 100644 --- a/functions/api/v1/timelines/public.ts +++ b/functions/api/v1/timelines/public.ts @@ -16,7 +16,7 @@ export const onRequest: PagesFunction = async ({ request, const only_media = searchParams.get('only_media') === 'true' const offset = Number.parseInt(searchParams.get('offset') ?? '0') const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), { local, remote, only_media, offset }) + return handleRequest(domain, await getDatabase(env), { local, remote, only_media, offset }) } export async function handleRequest( diff --git a/functions/api/v1/timelines/tag/[tag].ts b/functions/api/v1/timelines/tag/[tag].ts index 01a9654..9074fe4 100644 --- a/functions/api/v1/timelines/tag/[tag].ts +++ b/functions/api/v1/timelines/tag/[tag].ts @@ -11,7 +11,7 @@ const headers = { export const onRequest: PagesFunction = async ({ request, env, params }) => { const domain = new URL(request.url).hostname - return handleRequest(getDatabase(env), request, domain, params.tag as string) + return handleRequest(await getDatabase(env), request, domain, params.tag as string) } export async function handleRequest(db: Database, request: Request, domain: string, tag: string): Promise { diff --git a/functions/api/v2/instance.ts b/functions/api/v2/instance.ts index eea8c43..adc34b2 100644 --- a/functions/api/v2/instance.ts +++ b/functions/api/v2/instance.ts @@ -7,7 +7,7 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequest: PagesFunction = async ({ env, request }) => { const domain = new URL(request.url).hostname - return handleRequest(domain, getDatabase(env), env) + return handleRequest(domain, await getDatabase(env), env) } export async function handleRequest(domain: string, db: Database, env: Env) { diff --git a/functions/api/v2/media.ts b/functions/api/v2/media.ts index f19b3f5..f3c5bd5 100644 --- a/functions/api/v2/media.ts +++ b/functions/api/v2/media.ts @@ -9,7 +9,7 @@ import { mastodonIdSymbol } from 'wildebeest/backend/src/activitypub/objects' import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequestPost: PagesFunction = async ({ request, env, data }) => { - return handleRequestPost(request, getDatabase(env), data.connectedActor, env.CF_ACCOUNT_ID, env.CF_API_TOKEN) + return handleRequestPost(request, await getDatabase(env), data.connectedActor, env.CF_ACCOUNT_ID, env.CF_API_TOKEN) } export async function handleRequestPost( diff --git a/functions/api/v2/media/[id].ts b/functions/api/v2/media/[id].ts index 2d1fa03..8e40cde 100644 --- a/functions/api/v2/media/[id].ts +++ b/functions/api/v2/media/[id].ts @@ -14,7 +14,7 @@ import { updateObjectProperty } from 'wildebeest/backend/src/activitypub/objects import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequestPut: PagesFunction = async ({ params, env, request }) => { - return handleRequestPut(getDatabase(env), params.id as UUID, request) + return handleRequestPut(await getDatabase(env), params.id as UUID, request) } type UpdateMedia = { diff --git a/functions/api/v2/search.ts b/functions/api/v2/search.ts index 019e54c..622de8c 100644 --- a/functions/api/v2/search.ts +++ b/functions/api/v2/search.ts @@ -22,7 +22,7 @@ type SearchResult = { } export const onRequest: PagesFunction = async ({ request, env }) => { - return handleRequest(getDatabase(env), request) + return handleRequest(await getDatabase(env), request) } export async function handleRequest(db: Database, request: Request): Promise { diff --git a/functions/api/wb/settings/account/alias.ts b/functions/api/wb/settings/account/alias.ts index a8ef1b1..6eaa3ca 100644 --- a/functions/api/wb/settings/account/alias.ts +++ b/functions/api/wb/settings/account/alias.ts @@ -8,7 +8,7 @@ import * as errors from 'wildebeest/backend/src/errors' import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequestPost: PagesFunction = async ({ env, request, data }) => { - return handleRequestPost(getDatabase(env), request, data.connectedActor) + return handleRequestPost(await getDatabase(env), request, data.connectedActor) } type AddAliasRequest = { diff --git a/functions/first-login.ts b/functions/first-login.ts index 8d0c418..c175aec 100644 --- a/functions/first-login.ts +++ b/functions/first-login.ts @@ -9,7 +9,7 @@ import * as access from 'wildebeest/backend/src/access' import { type Database, getDatabase } from 'wildebeest/backend/src/database' export const onRequestPost: PagesFunction = async ({ request, env }) => { - return handlePostRequest(request, getDatabase(env), env.userKEK, env.ACCESS_AUTH_DOMAIN, env.ACCESS_AUD) + return handlePostRequest(request, await getDatabase(env), env.userKEK, env.ACCESS_AUTH_DOMAIN, env.ACCESS_AUD) } export async function handlePostRequest( diff --git a/functions/oauth/authorize.ts b/functions/oauth/authorize.ts index 7bda1c2..f0ed5bd 100644 --- a/functions/oauth/authorize.ts +++ b/functions/oauth/authorize.ts @@ -13,7 +13,7 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' const extractJWTFromRequest = (request: Request) => request.headers.get('Cf-Access-Jwt-Assertion') || '' export const onRequestPost: PagesFunction = async ({ request, env }) => { - return handleRequestPost(request, getDatabase(env), env.userKEK, env.ACCESS_AUTH_DOMAIN, env.ACCESS_AUD) + return handleRequestPost(request, await getDatabase(env), env.userKEK, env.ACCESS_AUTH_DOMAIN, env.ACCESS_AUD) } export async function buildRedirect( diff --git a/functions/oauth/token.ts b/functions/oauth/token.ts index e44e940..ffc7478 100644 --- a/functions/oauth/token.ts +++ b/functions/oauth/token.ts @@ -12,7 +12,7 @@ type Body = { } export const onRequest: PagesFunction = async ({ request, env }) => { - return handleRequest(getDatabase(env), request) + return handleRequest(await getDatabase(env), request) } export async function handleRequest(db: Database, request: Request): Promise { From 36a926e81661b57bb32f16f993bec8070db65a4e Mon Sep 17 00:00:00 2001 From: James Culveyhouse Date: Fri, 17 Feb 2023 14:58:44 -0600 Subject: [PATCH 02/62] Aliases page --- .../routes/(admin)/settings/aliases/index.tsx | 65 +++++++++++++++++++ .../src/routes/(admin)/settings/layout.tsx | 29 +++++++++ .../(admin)/settings/migration/index.tsx | 5 ++ 3 files changed, 99 insertions(+) create mode 100644 frontend/src/routes/(admin)/settings/aliases/index.tsx create mode 100644 frontend/src/routes/(admin)/settings/layout.tsx create mode 100644 frontend/src/routes/(admin)/settings/migration/index.tsx diff --git a/frontend/src/routes/(admin)/settings/aliases/index.tsx b/frontend/src/routes/(admin)/settings/aliases/index.tsx new file mode 100644 index 0000000..0bc874b --- /dev/null +++ b/frontend/src/routes/(admin)/settings/aliases/index.tsx @@ -0,0 +1,65 @@ +import { component$ } from '@builder.io/qwik' + +export default component$(() => { + return ( +
+

Account Aliases

+ +
+ Successfully created a new alias. You can now initiate the move from the old account. +
+ +

+ If you want to move from another account to this one, here you can create an alias, which is required before you + can proceed with moving followers from the old account to this one. This action by itself is harmless and + reversible. The account migration is initiated from the old account. +

+ +
+ +
Specify the username@domain of the account you want to move from
+
+ + + + + + + + + + + + + + + + + + +
Handle of the old account
test +
+ + Unlink Alias +
+
test 2 +
+ + Unlink Alias +
+
+
+ ) +}) diff --git a/frontend/src/routes/(admin)/settings/layout.tsx b/frontend/src/routes/(admin)/settings/layout.tsx new file mode 100644 index 0000000..5b0d8ff --- /dev/null +++ b/frontend/src/routes/(admin)/settings/layout.tsx @@ -0,0 +1,29 @@ +import { component$, Slot } from '@builder.io/qwik' +import { WildebeestLogo } from '~/components/MastodonLogo' + +export default component$(() => { + return ( +
+ +
+ +
+
+
+ ) +}) + +export const AccountSidebar = component$(() => { + return ( +
+
+
+ +
+ {/*
    +
  • Account
  • +
*/} +
+
+ ) +}) diff --git a/frontend/src/routes/(admin)/settings/migration/index.tsx b/frontend/src/routes/(admin)/settings/migration/index.tsx new file mode 100644 index 0000000..73bb0cf --- /dev/null +++ b/frontend/src/routes/(admin)/settings/migration/index.tsx @@ -0,0 +1,5 @@ +import { component$ } from '@builder.io/qwik' + +export default component$(() => { + return
TODO: migration
+}) From b16db6f510f21b1e70ca43937611a56f0de0518c Mon Sep 17 00:00:00 2001 From: James Culveyhouse Date: Thu, 23 Feb 2023 11:29:18 -0600 Subject: [PATCH 03/62] UI for migration page --- .../routes/(admin)/settings/aliases/index.tsx | 7 ++ .../(admin)/settings/migration/index.tsx | 86 ++++++++++++++++++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/frontend/src/routes/(admin)/settings/aliases/index.tsx b/frontend/src/routes/(admin)/settings/aliases/index.tsx index 0bc874b..7e90375 100644 --- a/frontend/src/routes/(admin)/settings/aliases/index.tsx +++ b/frontend/src/routes/(admin)/settings/aliases/index.tsx @@ -1,6 +1,13 @@ import { component$ } from '@builder.io/qwik' +import { loader$ } from '@builder.io/qwik-city' + +export const loader = loader$(({ redirect }) => { + redirect(303, '/explore') +}) export default component$(() => { + loader.use() + return (

Account Aliases

diff --git a/frontend/src/routes/(admin)/settings/migration/index.tsx b/frontend/src/routes/(admin)/settings/migration/index.tsx index 73bb0cf..52ab638 100644 --- a/frontend/src/routes/(admin)/settings/migration/index.tsx +++ b/frontend/src/routes/(admin)/settings/migration/index.tsx @@ -1,5 +1,89 @@ import { component$ } from '@builder.io/qwik' +import { loader$ } from '@builder.io/qwik-city' + +export const loader = loader$(({ redirect }) => { + redirect(303, '/explore') +}) export default component$(() => { - return
TODO: migration
+ loader.use() + + return ( +
+

Account Migration

+ +
Your account is not currently being redirected to any other account.
+ +

Move to a different account

+ +

Before proceeding, please read these notes carefully:

+ +
    +
  • This action will move all followers from the current account to the new account
  • +
  • + Your current account's profile will be updated with a redirect notice and be excluded from searches +
  • +
  • No other data will be moved automatically
  • +
  • The new account must first be configured to back-reference this one
  • +
  • After moving there is a waiting period during which you will not be able to move again
  • +
  • + Your current account will not be fully usable afterwards. However, you will have access to data export as well + as re-activation. +
  • +
+ +

+ Alternatively, you can only put up a redirect on your profile. +

+ +
+
+
+ +
+ Specify the username@domain of the account you want to move to +
+
+ +
+
+
+ +
+ For security purposes please enter the password of the current account +
+
+ +
+
+ + + +

Moving from a different account

+ +

+ To move from another account to this one, first you need to{' '} + create an account alias. +

+
+ ) }) From 4f71caac31c6808c0e6c20797509186add49bf0a Mon Sep 17 00:00:00 2001 From: James Culveyhouse Date: Mon, 27 Feb 2023 16:01:48 -0600 Subject: [PATCH 04/62] Sign in button and authed account settings page --- .../layout/RightColumn/RightColumn.tsx | 17 ++++++++++ .../routes/(admin)/settings/aliases/index.tsx | 16 ++++++---- .../src/routes/(admin)/settings/index.tsx | 6 ++++ .../src/routes/(admin)/settings/layout.tsx | 19 ++++++++++-- .../(admin)/settings/migration/index.tsx | 16 ++++++---- frontend/src/routes/layout.tsx | 31 +++++++++++++++++++ frontend/src/types.ts | 5 +++ frontend/src/utils/checkAuth.ts | 29 +++++++++++++++++ package.json | 4 +-- 9 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 frontend/src/routes/(admin)/settings/index.tsx create mode 100644 frontend/src/routes/layout.tsx create mode 100644 frontend/src/utils/checkAuth.ts diff --git a/frontend/src/components/layout/RightColumn/RightColumn.tsx b/frontend/src/components/layout/RightColumn/RightColumn.tsx index 1443b2f..af3f742 100644 --- a/frontend/src/components/layout/RightColumn/RightColumn.tsx +++ b/frontend/src/components/layout/RightColumn/RightColumn.tsx @@ -1,6 +1,7 @@ import { component$ } from '@builder.io/qwik' import { Link, useLocation } from '@builder.io/qwik-city' import { WildebeestLogo } from '~/components/MastodonLogo' +import { accessLoader } from '~/routes/layout' type LinkConfig = { iconName: string @@ -10,6 +11,7 @@ type LinkConfig = { } export default component$(() => { + const accessData = accessLoader.use().value const location = useLocation() const renderNavLink = ({ iconName, linkText, linkTarget, linkActiveRegex }: LinkConfig) => { @@ -52,6 +54,21 @@ export default component$(() => {
{renderNavLink(aboutLink)}
*/} + + {!accessData.isAuthorized && ( + + Sign in + + )} + {accessData.isAuthorized && ( + + + Preferences + + )}
) diff --git a/frontend/src/routes/(admin)/settings/aliases/index.tsx b/frontend/src/routes/(admin)/settings/aliases/index.tsx index 7e90375..1d28fd6 100644 --- a/frontend/src/routes/(admin)/settings/aliases/index.tsx +++ b/frontend/src/routes/(admin)/settings/aliases/index.tsx @@ -1,13 +1,17 @@ import { component$ } from '@builder.io/qwik' import { loader$ } from '@builder.io/qwik-city' +import { WildebeestEnv } from '~/types' +import { checkAuth } from '~/utils/checkAuth' -export const loader = loader$(({ redirect }) => { - redirect(303, '/explore') +export const loader = loader$(async ({ request, platform, redirect }) => { + const isAuthorized = await checkAuth(request, platform) + + if (!isAuthorized) { + redirect(303, '/explore') + } }) export default component$(() => { - loader.use() - return (

Account Aliases

@@ -36,9 +40,9 @@ export default component$(() => { /> diff --git a/frontend/src/routes/(admin)/settings/index.tsx b/frontend/src/routes/(admin)/settings/index.tsx new file mode 100644 index 0000000..9c935b2 --- /dev/null +++ b/frontend/src/routes/(admin)/settings/index.tsx @@ -0,0 +1,6 @@ +import { component$ } from '@builder.io/qwik' + +export default component$(() => { + // In the future, a settings homepage will be here + return
+}) diff --git a/frontend/src/routes/(admin)/settings/layout.tsx b/frontend/src/routes/(admin)/settings/layout.tsx index 5b0d8ff..9ff3fde 100644 --- a/frontend/src/routes/(admin)/settings/layout.tsx +++ b/frontend/src/routes/(admin)/settings/layout.tsx @@ -20,9 +20,22 @@ export const AccountSidebar = component$(() => {
- {/*
    -
  • Account
  • -
*/} + + + Back to Wildebeest + + ) diff --git a/frontend/src/routes/(admin)/settings/migration/index.tsx b/frontend/src/routes/(admin)/settings/migration/index.tsx index 52ab638..624aad3 100644 --- a/frontend/src/routes/(admin)/settings/migration/index.tsx +++ b/frontend/src/routes/(admin)/settings/migration/index.tsx @@ -1,13 +1,17 @@ import { component$ } from '@builder.io/qwik' import { loader$ } from '@builder.io/qwik-city' +import { WildebeestEnv } from '~/types' +import { checkAuth } from '~/utils/checkAuth' -export const loader = loader$(({ redirect }) => { - redirect(303, '/explore') +export const loader = loader$(async ({ request, platform, redirect }) => { + const isAuthorized = await checkAuth(request, platform) + + if (!isAuthorized) { + redirect(303, '/explore') + } }) export default component$(() => { - loader.use() - return (

Account Migration

@@ -73,9 +77,9 @@ export default component$(() => {

Moving from a different account

diff --git a/frontend/src/routes/layout.tsx b/frontend/src/routes/layout.tsx new file mode 100644 index 0000000..e99721a --- /dev/null +++ b/frontend/src/routes/layout.tsx @@ -0,0 +1,31 @@ +import { component$, Slot } from '@builder.io/qwik' +import { loader$ } from '@builder.io/qwik-city' +import * as access from 'wildebeest/backend/src/access' +import { WildebeestEnv } from '~/types' +import { checkAuth } from '~/utils/checkAuth' + +type AccessLoaderData = { + loginUrl: string + isAuthorized: boolean +} + +export const accessLoader = loader$>(async ({ platform, request }) => { + const isAuthorized = await checkAuth(request, platform) + + return { + isAuthorized, + loginUrl: access.generateLoginURL({ + redirectURL: request.url, + domain: platform.ACCESS_AUTH_DOMAIN, + aud: platform.ACCESS_AUD, + }), + } +}) + +export default component$(() => { + return ( + <> + + + ) +}) diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 73ed0dc..ab9d88f 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -216,3 +216,8 @@ export type History = { accounts: string uses: string } + +export type WildebeestEnv = { + ACCESS_AUTH_DOMAIN: string + ACCESS_AUD: string +} diff --git a/frontend/src/utils/checkAuth.ts b/frontend/src/utils/checkAuth.ts new file mode 100644 index 0000000..23fbe14 --- /dev/null +++ b/frontend/src/utils/checkAuth.ts @@ -0,0 +1,29 @@ +import { RequestContext } from '@builder.io/qwik-city/middleware/request-handler' +import * as access from 'wildebeest/backend/src/access' + +type Env = { + ACCESS_AUTH_DOMAIN: string + ACCESS_AUD: string +} + +export const checkAuth = async (request: RequestContext, platform: Env) => { + const jwt = request.headers.get('Cf-Access-Jwt-Assertion') || '' + if (!jwt) return false + + try { + const validate = access.generateValidator({ + jwt, + domain: platform.ACCESS_AUTH_DOMAIN, + aud: platform.ACCESS_AUD, + }) + await validate(new Request(request.url)) + } catch { + return false + } + + const identity = await access.getIdentity({ jwt, domain: platform.ACCESS_AUTH_DOMAIN }) + if (identity) { + return true + } + return false +} diff --git a/package.json b/package.json index 0f4ed09..82315a8 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,8 @@ "pages": "NO_D1_WARNING=true wrangler pages", "database:migrate": "yarn d1 migrations apply DATABASE", "database:create-mock": "rm -f .wrangler/state/d1/DATABASE.sqlite3 && CI=true yarn database:migrate --local && node ./frontend/mock-db/run.mjs", - "dev": "export COMMIT_HASH=$(git rev-parse HEAD) && yarn build && yarn database:migrate --local && yarn pages dev frontend/dist --d1 DATABASE --persist --compatibility-date=2022-12-20 --binding 'INSTANCE_TITLE=Test Wildebeest' 'INSTANCE_DESCR=My Wildebeest Instance' --live-reload", - "ci-dev-test-ui": "yarn build && yarn database:create-mock && yarn pages dev frontend/dist --d1 DATABASE --persist --port 8788 --binding 'INSTANCE_TITLE=Test Wildebeest' 'INSTANCE_DESCR=My Wildebeest Instance' --compatibility-date=2022-12-20", + "dev": "export COMMIT_HASH=$(git rev-parse HEAD) && yarn build && yarn database:migrate --local && yarn pages dev frontend/dist --d1 DATABASE --persist --compatibility-date=2022-12-20 --binding 'INSTANCE_TITLE=Test Wildebeest' 'INSTANCE_DESCR=My Wildebeest Instance' 'ACCESS_AUTH_DOMAIN=0.0.0.0.cloudflareaccess.com' 'ACCESS_AUD=DEV_AUD' --live-reload", + "ci-dev-test-ui": "yarn build && yarn database:create-mock && yarn pages dev frontend/dist --d1 DATABASE --persist --port 8788 --binding 'INSTANCE_TITLE=Test Wildebeest' 'INSTANCE_DESCR=My Wildebeest Instance' 'ACCESS_AUTH_DOMAIN=0.0.0.0.cloudflareaccess.com' 'ACCESS_AUD=DEV_AUD' --compatibility-date=2022-12-20", "deploy:init": "yarn pages project create wildebeest && yarn d1 create wildebeest", "deploy": "yarn build && yarn database:migrate && yarn pages publish frontend/dist --project-name=wildebeest" }, From d9cc274673bfc917ce017a1b1b3b8eb07c2da233 Mon Sep 17 00:00:00 2001 From: James Culveyhouse Date: Tue, 28 Feb 2023 00:22:19 -0600 Subject: [PATCH 05/62] Integration alias api endpoint --- .../routes/(admin)/settings/aliases/index.tsx | 148 +++++++++++------- .../src/routes/(admin)/settings/layout.tsx | 6 +- .../(admin)/settings/migration/index.tsx | 21 ++- 3 files changed, 106 insertions(+), 69 deletions(-) diff --git a/frontend/src/routes/(admin)/settings/aliases/index.tsx b/frontend/src/routes/(admin)/settings/aliases/index.tsx index 1d28fd6..4a5d1e3 100644 --- a/frontend/src/routes/(admin)/settings/aliases/index.tsx +++ b/frontend/src/routes/(admin)/settings/aliases/index.tsx @@ -1,4 +1,4 @@ -import { component$ } from '@builder.io/qwik' +import { component$, useStore, useSignal, $ } from '@builder.io/qwik' import { loader$ } from '@builder.io/qwik-city' import { WildebeestEnv } from '~/types' import { checkAuth } from '~/utils/checkAuth' @@ -12,65 +12,97 @@ export const loader = loader$(async ({ request, platform, r }) export default component$(() => { + const ref = useSignal() + const state = useStore({ alias: '' }) + const toast = useSignal<'success' | 'failure' | null>(null) + + const handleInput = $((event: Event) => { + state.alias = (event.target as HTMLInputElement).value + }) + + const handleSubmit = $(async () => { + const res = await fetch('/api/wb/settings/account/alias', { method: 'POST', body: JSON.stringify(state) }) + if (res.status == 200) { + toast.value = 'success' + } else { + toast.value = 'failure' + } + }) + return ( -
-

Account Aliases

+ +
+

Account Aliases

-
- Successfully created a new alias. You can now initiate the move from the old account. + {toast.value === 'success' && ( +
+ Successfully created a new alias. You can now initiate the move from the old account. +
+ )} + + {toast.value === 'failure' && ( +
+ Failed to create alias. +
+ )} + +

+ If you want to move from another account to this one, here you can create an alias, which is required before + you can proceed with moving followers from the old account to this one. This action by itself is harmless and + reversible. The account migration is initiated from the old account. +

+ +
+ +
+ Specify the username@domain of the account you want to move from +
+
+ + + + {/*
+ + + + + + + + + + + + + + +
Handle of the old account
test +
+ + Unlink Alias +
+
test 2 +
+ + Unlink Alias +
+
*/}
- -

- If you want to move from another account to this one, here you can create an alias, which is required before you - can proceed with moving followers from the old account to this one. This action by itself is harmless and - reversible. The account migration is initiated from the old account. -

- -
- -
Specify the username@domain of the account you want to move from
-
- - - - - - - - - - - - - - - - - - -
Handle of the old account
test -
- - Unlink Alias -
-
test 2 -
- - Unlink Alias -
-
- + ) }) diff --git a/frontend/src/routes/(admin)/settings/layout.tsx b/frontend/src/routes/(admin)/settings/layout.tsx index 9ff3fde..db131f0 100644 --- a/frontend/src/routes/(admin)/settings/layout.tsx +++ b/frontend/src/routes/(admin)/settings/layout.tsx @@ -3,7 +3,7 @@ import { WildebeestLogo } from '~/components/MastodonLogo' export default component$(() => { return ( -
+
-
From 74aa6656ca66252d9be38974b701358d0f851d70 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 28 Feb 2023 11:08:51 +0000 Subject: [PATCH 06/62] add missing argument to handleRequestGet --- frontend/src/routes/(frontend)/[accountId]/layout.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/src/routes/(frontend)/[accountId]/layout.tsx b/frontend/src/routes/(frontend)/[accountId]/layout.tsx index 109b766..18a8b80 100644 --- a/frontend/src/routes/(frontend)/[accountId]/layout.tsx +++ b/frontend/src/routes/(frontend)/[accountId]/layout.tsx @@ -12,6 +12,7 @@ import { getDocumentHead } from '~/utils/getDocumentHead' import * as statusAPI from 'wildebeest/functions/api/v1/statuses/[id]' import { useAccountUrl } from '~/utils/useAccountUrl' import { getDatabase } from 'wildebeest/backend/src/database' +import { Person } from 'wildebeest/backend/src/activitypub/actors' export const accountPageLoader = loader$< Promise<{ account: MastodonAccount; accountHandle: string; isValidStatus: boolean }>, @@ -25,7 +26,12 @@ export const accountPageLoader = loader$< const accountId = url.pathname.split('/')[1] try { - const statusResponse = await statusAPI.handleRequestGet(await getDatabase(platform), params.statusId, domain) + const statusResponse = await statusAPI.handleRequestGet( + await getDatabase(platform), + params.statusId, + domain, + null as unknown as Person + ) const statusText = await statusResponse.text() isValidStatus = !!statusText } catch { From 55c780ad02db9dfed19991fb5bdfe75339c2707f Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 28 Feb 2023 11:42:52 +0000 Subject: [PATCH 07/62] fix DocumentHeadValue being modified as readonly --- frontend/src/utils/getDocumentHead.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/utils/getDocumentHead.ts b/frontend/src/utils/getDocumentHead.ts index 36b9b9a..915bd5b 100644 --- a/frontend/src/utils/getDocumentHead.ts +++ b/frontend/src/utils/getDocumentHead.ts @@ -10,6 +10,8 @@ type DocumentHeadData = { } } +type NoReadonly = { -readonly [P in keyof T]: NoReadonly } + /** * Generates a head to provide to QwikCity * @@ -18,7 +20,7 @@ type DocumentHeadData = { * @returns the QwikCity head ready to use */ export function getDocumentHead(data: DocumentHeadData, head?: DocumentHeadValue) { - const result: DocumentHeadValue = { meta: [] } + const result: NoReadonly = { meta: [] } const setMeta = (name: string, content: string) => { if (head?.meta?.some((meta) => meta.name === name)) { From 47ce3a6e898cda345ce4f156e193186cf05d1c36 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 28 Feb 2023 13:59:35 +0000 Subject: [PATCH 08/62] add add_admin migration --- migrations/0009_add_admin.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 migrations/0009_add_admin.sql diff --git a/migrations/0009_add_admin.sql b/migrations/0009_add_admin.sql new file mode 100644 index 0000000..d470755 --- /dev/null +++ b/migrations/0009_add_admin.sql @@ -0,0 +1,10 @@ +-- Migration number: 0009 2023-02-28T13:58:08.319Z + +ALTER TABLE actors + ADD is_admin INTEGER; + +UPDATE actors SET is_admin = 1 +WHERE id = + (SELECT id + FROM actors + ORDER BY cdate ASC LIMIT 1 ); From 30a817563e1e049cc501867fb63a02db08456972 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 24 Feb 2023 10:47:35 +0000 Subject: [PATCH 09/62] remove deprecated .use() loader calls --- frontend/src/routes/(admin)/settings/migration/index.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/routes/(admin)/settings/migration/index.tsx b/frontend/src/routes/(admin)/settings/migration/index.tsx index 56c793c..9998c90 100644 --- a/frontend/src/routes/(admin)/settings/migration/index.tsx +++ b/frontend/src/routes/(admin)/settings/migration/index.tsx @@ -15,6 +15,8 @@ export const loader = loader$(async ({ redirect }) => { }) export default component$(() => { + loader() + return (

Account Migration

From c184d28e4f530fd788a0ef618a86dcbe8bd797d1 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 24 Feb 2023 11:00:49 +0000 Subject: [PATCH 10/62] improve dev/todo loaders --- frontend/src/routes/(admin)/settings/aliases/index.tsx | 2 +- frontend/src/routes/(admin)/settings/layout.tsx | 9 +++++++++ frontend/src/routes/(admin)/settings/migration/index.tsx | 2 -- frontend/src/routes/(frontend)/about/index.tsx | 7 ++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/routes/(admin)/settings/aliases/index.tsx b/frontend/src/routes/(admin)/settings/aliases/index.tsx index 4a5d1e3..decfc0a 100644 --- a/frontend/src/routes/(admin)/settings/aliases/index.tsx +++ b/frontend/src/routes/(admin)/settings/aliases/index.tsx @@ -3,7 +3,7 @@ import { loader$ } from '@builder.io/qwik-city' import { WildebeestEnv } from '~/types' import { checkAuth } from '~/utils/checkAuth' -export const loader = loader$(async ({ request, platform, redirect }) => { +export const loader = loader$(async ({ request, platform, redirect }) => { const isAuthorized = await checkAuth(request, platform) if (!isAuthorized) { diff --git a/frontend/src/routes/(admin)/settings/layout.tsx b/frontend/src/routes/(admin)/settings/layout.tsx index db131f0..2f9b6e1 100644 --- a/frontend/src/routes/(admin)/settings/layout.tsx +++ b/frontend/src/routes/(admin)/settings/layout.tsx @@ -1,7 +1,16 @@ import { component$, Slot } from '@builder.io/qwik' import { WildebeestLogo } from '~/components/MastodonLogo' +import { loader$ } from '@builder.io/qwik-city' +import { getNotFoundHtml } from '~/utils/getNotFoundHtml/getNotFoundHtml' + +export const loader = loader$(({ html }) => { + html(404, getNotFoundHtml()) +}) + export default component$(() => { + loader() + return (
diff --git a/frontend/src/routes/(admin)/settings/migration/index.tsx b/frontend/src/routes/(admin)/settings/migration/index.tsx index 9998c90..56c793c 100644 --- a/frontend/src/routes/(admin)/settings/migration/index.tsx +++ b/frontend/src/routes/(admin)/settings/migration/index.tsx @@ -15,8 +15,6 @@ export const loader = loader$(async ({ redirect }) => { }) export default component$(() => { - loader() - return (

Account Migration

diff --git a/frontend/src/routes/(frontend)/about/index.tsx b/frontend/src/routes/(frontend)/about/index.tsx index 891bcf4..d24a8b9 100644 --- a/frontend/src/routes/(frontend)/about/index.tsx +++ b/frontend/src/routes/(frontend)/about/index.tsx @@ -7,6 +7,7 @@ import { HtmlContent } from '~/components/HtmlContent/HtmlContent' import { george } from '~/dummyData/accounts' import { Account } from '~/types' import { getDocumentHead } from '~/utils/getDocumentHead' +import { getNotFoundHtml } from '~/utils/getNotFoundHtml/getNotFoundHtml' import { instanceLoader } from '../layout' type AboutInfo = { @@ -23,9 +24,9 @@ type AboutInfo = { } } -export const aboutInfoLoader = loader$>(async ({ resolveValue, request, redirect }) => { - // TODO: properly implement loader and remove redirect - throw redirect(302, '/') +export const aboutInfoLoader = loader$>(async ({ resolveValue, request, html }) => { + // TODO: properly implement loader and remove the following 404 throw + throw html(404, getNotFoundHtml()) const instance = await resolveValue(instanceLoader) return { From 3d39ca74c5688e354e80f9276da518b89bb3ae21 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 24 Feb 2023 15:50:12 +0000 Subject: [PATCH 11/62] remove unneeded top level concurrently dependency --- package.json | 1 - yarn.lock | 53 +++------------------------------------------------- 2 files changed, 3 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index 82315a8..ce8b5d9 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,6 @@ "@typescript-eslint/eslint-plugin": "^5.46.1", "@typescript-eslint/parser": "^5.46.1", "better-sqlite3": "8", - "concurrently": "^7.6.0", "eslint": "^8.29.0", "jest": "^29.3.1", "jest-environment-miniflare": "^2.11.0", diff --git a/yarn.lock b/yarn.lock index 0e7addd..ac6621e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1369,7 +1369,7 @@ chalk@^2.0.0, chalk@^2.4.1: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.1.0: +chalk@^4.0.0: version "4.1.2" resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -1460,21 +1460,6 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -concurrently@^7.6.0: - version "7.6.0" - resolved "https://registry.yarnpkg.com/concurrently/-/concurrently-7.6.0.tgz#531a6f5f30cf616f355a4afb8f8fcb2bba65a49a" - integrity sha512-BKtRgvcJGeZ4XttiDiNcFiRlxoAeZOseqUvyYRUp/Vtd+9p1ULmeoSqGsDA+2ivdeDFpqrJvGvmI+StKfKl5hw== - dependencies: - chalk "^4.1.0" - date-fns "^2.29.1" - lodash "^4.17.21" - rxjs "^7.0.0" - shell-quote "^1.7.3" - spawn-command "^0.0.2-1" - supports-color "^8.1.0" - tree-kill "^1.2.2" - yargs "^17.3.1" - convert-source-map@^1.6.0, convert-source-map@^1.7.0: version "1.9.0" resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz" @@ -1520,11 +1505,6 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" -date-fns@^2.29.1: - version "2.29.3" - resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.3.tgz#27402d2fc67eb442b511b70bbdf98e6411cd68a8" - integrity sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA== - debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" @@ -3090,11 +3070,6 @@ lodash.merge@^4.6.2: resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash@^4.17.21: - version "4.17.21" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" - integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== - lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz" @@ -3721,13 +3696,6 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -rxjs@^7.0.0: - version "7.8.0" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.8.0.tgz#90a938862a82888ff4c7359811a595e14e1e09a4" - integrity sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg== - dependencies: - tslib "^2.1.0" - safe-buffer@^5.0.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" @@ -3800,7 +3768,7 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shell-quote@^1.6.1, shell-quote@^1.7.3: +shell-quote@^1.6.1: version "1.7.4" resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.4.tgz#33fe15dee71ab2a81fcbd3a52106c5cfb9fb75d8" integrity sha512-8o/QEhSSRb1a5i7TFR0iM4G16Z0vYB2OQVs4G3aAFXjn3T6yEx8AZxy1PgDF7I00LZHYA3WxaSYIf5e5sAX8Rw== @@ -3874,11 +3842,6 @@ sourcemap-codec@^1.4.8: resolved "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz" integrity sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA== -spawn-command@^0.0.2-1: - version "0.0.2-1" - resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2-1.tgz#62f5e9466981c1b796dc5929937e11c9c6921bd0" - integrity sha512-n98l9E2RMSJ9ON1AKisHzz7V42VDiBQGY6PB1BwRglz99wpVsSuGzQ+jOi6lFXBGVTCrRpltvjm+/XA+tpeJrg== - spdx-correct@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" @@ -4029,7 +3992,7 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" -supports-color@^8.0.0, supports-color@^8.1.0: +supports-color@^8.0.0: version "8.1.1" resolved "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz" integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== @@ -4102,11 +4065,6 @@ toucan-js@^3.1.0: "@sentry/types" "7.28.1" "@sentry/utils" "7.28.1" -tree-kill@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" - integrity sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A== - ts-jest@^29.0.3: version "29.0.3" resolved "https://registry.npmjs.org/ts-jest/-/ts-jest-29.0.3.tgz" @@ -4126,11 +4084,6 @@ tslib@^1.8.1, tslib@^1.9.3: resolved "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2.1.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e" - integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA== - tsutils@^3.21.0: version "3.21.0" resolved "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz" From da893972098f40ed40fec5a809aea5f61182860f Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 24 Feb 2023 15:51:28 +0000 Subject: [PATCH 12/62] improve frontend watch script and remove no longer needed concurrently dependency --- frontend/package.json | 4 +-- frontend/yarn.lock | 83 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index dd75947..d3b2f25 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "lint": "eslint src mock-db adaptors", "build": "vite build && vite build -c adaptors/cloudflare-pages/vite.config.ts", "dev": "vite --mode ssr", - "watch": "concurrently \"vite build -w\" \"vite build -w -c adaptors/cloudflare-pages/vite.config.ts\"" + "watch": "nodemon -w ./src --ext tsx,ts --exec npm run build" }, "devDependencies": { "@builder.io/qwik": "0.18.1", @@ -22,12 +22,12 @@ "@typescript-eslint/eslint-plugin": "5.46.1", "@typescript-eslint/parser": "5.46.1", "autoprefixer": "10.4.11", - "concurrently": "^7.6.0", "eslint": "8.30.0", "eslint-plugin-qwik": "0.16.1", "jest": "^29.3.1", "lorem-ipsum": "^2.0.8", "node-fetch": "3.3.0", + "nodemon": "^2.0.20", "postcss": "^8.4.16", "prettier": "2.8.1", "sass": "^1.57.0", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index cc93c0f..1599b1a 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1031,6 +1031,11 @@ "@typescript-eslint/types" "5.46.1" eslint-visitor-keys "^3.3.0" +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== + acorn-jsx@^5.0.0, acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" @@ -1356,7 +1361,7 @@ character-reference-invalid@^2.0.0: resolved "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz" integrity sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw== -"chokidar@>=3.0.0 <4.0.0", chokidar@^3.5.3: +"chokidar@>=3.0.0 <4.0.0", chokidar@^3.5.2, chokidar@^3.5.3: version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -1500,6 +1505,13 @@ date-fns@^2.29.1: resolved "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz" integrity sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA== +debug@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" @@ -2149,6 +2161,11 @@ human-signals@^2.1.0: resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +ignore-by-default@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09" + integrity sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA== + ignore@^5.2.0: version "5.2.4" resolved "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz" @@ -3311,6 +3328,11 @@ ms@2.1.2: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + mz@^2.7.0: version "2.7.0" resolved "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz" @@ -3359,6 +3381,29 @@ node-releases@^2.0.6: resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.8.tgz" integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A== +nodemon@^2.0.20: + version "2.0.20" + resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.20.tgz#e3537de768a492e8d74da5c5813cb0c7486fc701" + integrity sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw== + dependencies: + chokidar "^3.5.2" + debug "^3.2.7" + ignore-by-default "^1.0.1" + minimatch "^3.1.2" + pstree.remy "^1.1.8" + semver "^5.7.1" + simple-update-notifier "^1.0.7" + supports-color "^5.5.0" + touch "^3.1.0" + undefsafe "^2.0.5" + +nopt@~1.0.10: + version "1.0.10" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee" + integrity sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg== + dependencies: + abbrev "1" + normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz" @@ -3621,6 +3666,11 @@ property-information@^6.0.0: resolved "https://registry.npmjs.org/property-information/-/property-information-6.2.0.tgz" integrity sha512-kma4U7AFCTwpqq5twzC1YVIDXSqg6qQK6JN0smOw8fgRy1OkMi0CYSzFmsy6dnqSenamAtj0CyXMUJ1Mf6oROg== +pstree.remy@^1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a" + integrity sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w== + punycode@^2.1.0: version "2.1.1" resolved "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" @@ -3790,11 +3840,21 @@ semver@7.x, semver@^7.3.5, semver@^7.3.7: dependencies: lru-cache "^6.0.0" +semver@^5.7.1: + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + semver@^6.0.0, semver@^6.3.0: version "6.3.0" resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@~7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" + integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== + shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz" @@ -3817,6 +3877,13 @@ signal-exit@^3.0.3, signal-exit@^3.0.7: resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== +simple-update-notifier@^1.0.7: + version "1.1.0" + resolved "https://registry.yarnpkg.com/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz#67694c121de354af592b347cdba798463ed49c82" + integrity sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg== + dependencies: + semver "~7.0.0" + sisteransi@^1.0.5: version "1.0.5" resolved "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz" @@ -3948,7 +4015,7 @@ sucrase@^3.20.3: pirates "^4.0.1" ts-interface-checker "^0.1.9" -supports-color@^5.3.0: +supports-color@^5.3.0, supports-color@^5.5.0: version "5.5.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== @@ -4048,6 +4115,13 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +touch@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.0.tgz#fe365f5f75ec9ed4e56825e0bb76d24ab74af83b" + integrity sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA== + dependencies: + nopt "~1.0.10" + tree-kill@^1.2.2: version "1.2.2" resolved "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz" @@ -4135,6 +4209,11 @@ typescript@4.9.4: resolved "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz" integrity sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg== +undefsafe@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" + integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== + undici@5.19.1: version "5.19.1" resolved "https://registry.yarnpkg.com/undici/-/undici-5.19.1.tgz#92b1fd3ab2c089b5a6bd3e579dcda8f1934ebf6d" From b131e83aa0cf8514f2d3b1380c760a0153de6770 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 27 Feb 2023 20:13:44 +0000 Subject: [PATCH 13/62] bump qwik and qwikcity --- frontend/package.json | 4 ++-- frontend/yarn.lock | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index d3b2f25..48d8ceb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,8 +13,8 @@ "watch": "nodemon -w ./src --ext tsx,ts --exec npm run build" }, "devDependencies": { - "@builder.io/qwik": "0.18.1", - "@builder.io/qwik-city": "0.2.1", + "@builder.io/qwik": "0.19.2", + "@builder.io/qwik-city": "0.4.0", "@types/eslint": "8.4.10", "@types/jest": "^29.2.4", "@types/node": "^18.11.16", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 1599b1a..e7405e0 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -297,10 +297,10 @@ resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@builder.io/qwik-city@0.2.1": - version "0.2.1" - resolved "https://registry.yarnpkg.com/@builder.io/qwik-city/-/qwik-city-0.2.1.tgz#c57f481a75534ff54ddb0f38403acc66b5d02f41" - integrity sha512-g+ZC4Neo1XYQ/8uquUp6GKwr0eagpuCyQ3LkAtFhaIARaO67+cZfR6EFLJzf9wz5AVSt8/0QSD7wJEpni1i4IA== +"@builder.io/qwik-city@0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@builder.io/qwik-city/-/qwik-city-0.4.0.tgz#9afb97ba0e11119e44a2527f545e0d84a8fe9759" + integrity sha512-XNpmHzSHam7ZYrd12kJdwFerMEck0iOk3Wgb9IlVIuaN/nLuN033qrNWLVq+ZzlhplUea9DGc4job8qMix7WWA== dependencies: "@mdx-js/mdx" "2.3.0" "@types/mdx" "2.0.3" @@ -308,10 +308,10 @@ vfile "5.3.7" zod "^3.20.6" -"@builder.io/qwik@0.18.1": - version "0.18.1" - resolved "https://registry.yarnpkg.com/@builder.io/qwik/-/qwik-0.18.1.tgz#341d01c5749a07230c700a5e4df859b857654cd0" - integrity sha512-11qx5Wh6WRxgvHDJDppJORhykzkACUYuu9qRKEGdS3vTkBQ2Rr8NFDjYon2x6+8Wu9WukHk84ANywWnS91gS/w== +"@builder.io/qwik@0.19.2": + version "0.19.2" + resolved "https://registry.yarnpkg.com/@builder.io/qwik/-/qwik-0.19.2.tgz#d2f7d39bf6adb6de8497690cb3c41ad62a5fc1c6" + integrity sha512-Rxdyx96ucx0+nABLsg1sH7SgrlMdHgpxbR+NR3c53Ux4Jj+Xf+QHNQSIKF9zTV8KClrzmDlqILgmkDU4uMTVcg== "@cush/relative@^1.0.0": version "1.0.0" From 39636f5b99e6767ff4c635d1d064282384138426 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Tue, 28 Feb 2023 16:32:30 +0000 Subject: [PATCH 14/62] change remaining D1Database types --- backend/src/accounts/getAccount.ts | 2 +- backend/src/webfinger/index.ts | 3 ++- functions/api/v1/apps/verify_credentials.ts | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/src/accounts/getAccount.ts b/backend/src/accounts/getAccount.ts index 304ba8a..959e3fc 100644 --- a/backend/src/accounts/getAccount.ts +++ b/backend/src/accounts/getAccount.ts @@ -24,7 +24,7 @@ export async function getAccount(domain: string, accountId: string, db: Database } } -async function getRemoteAccount(handle: Handle, acct: string, db: D1Database): Promise { +async function getRemoteAccount(handle: Handle, acct: string, db: Database): Promise { // TODO: using webfinger isn't the optimal implementation. We could cache // the object in D1 and directly query the remote API, indicated by the actor's // url field. For now, let's keep it simple. diff --git a/backend/src/webfinger/index.ts b/backend/src/webfinger/index.ts index e0f8502..5fd011b 100644 --- a/backend/src/webfinger/index.ts +++ b/backend/src/webfinger/index.ts @@ -1,4 +1,5 @@ import * as actors from '../activitypub/actors' +import { type Database } from 'wildebeest/backend/src/database' import type { Actor } from '../activitypub/actors' export type WebFingerResponse = { @@ -11,7 +12,7 @@ const headers = { accept: 'application/jrd+json', } -export async function queryAcct(domain: string, db: D1Database, acct: string): Promise { +export async function queryAcct(domain: string, db: Database, acct: string): Promise { const url = await queryAcctLink(domain, acct) if (url === null) { return null diff --git a/functions/api/v1/apps/verify_credentials.ts b/functions/api/v1/apps/verify_credentials.ts index 9cbae78..e229d56 100644 --- a/functions/api/v1/apps/verify_credentials.ts +++ b/functions/api/v1/apps/verify_credentials.ts @@ -1,5 +1,6 @@ // https://docs.joinmastodon.org/methods/apps/#verify_credentials +import { type Database } from 'wildebeest/backend/src/database' import { cors } from 'wildebeest/backend/src/utils/cors' import { VAPIDPublicKey } from 'wildebeest/backend/src/mastodon/subscription' import { getVAPIDKeys } from 'wildebeest/backend/src/config' @@ -24,7 +25,7 @@ export const onRequest: PagesFunction = async ({ request, return handleRequest(env.DATABASE, request, getVAPIDKeys(env)) } -export async function handleRequest(db: D1Database, request: Request, vapidKeys: JWK) { +export async function handleRequest(db: Database, request: Request, vapidKeys: JWK) { if (request.method !== 'GET') { return new Response('', { status: 400 }) } From 217ff34353f36eacb4530e86ae81799e725da0ca Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Tue, 28 Feb 2023 16:51:38 +0000 Subject: [PATCH 15/62] introduce simple SQL query builder --- backend/src/database/d1.ts | 27 +++++++++++++++++++--- backend/src/database/index.ts | 8 +++++++ backend/src/mastodon/timeline.ts | 12 +++++----- backend/test/mastodon/apps.spec.ts | 2 ++ backend/test/mastodon/statuses.spec.ts | 4 ++-- backend/test/utils.ts | 4 +++- functions/api/v1/accounts/[id]/statuses.ts | 4 ++-- 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/backend/src/database/d1.ts b/backend/src/database/d1.ts index 99359e8..41c7fe4 100644 --- a/backend/src/database/d1.ts +++ b/backend/src/database/d1.ts @@ -1,6 +1,27 @@ -import { type Database } from 'wildebeest/backend/src/database' +import { type Database, QueryBuilder } from 'wildebeest/backend/src/database' import type { Env } from 'wildebeest/backend/src/types/env' -export default function make({ DATABASE }: Pick): Database { - return DATABASE +const qb: QueryBuilder = { + jsonExtract(obj: string, prop: string): string { + return `json_extract(${obj}, '$.${prop}')` + }, + + jsonExtractIsNull(obj: string, prop: string): string { + return `${qb.jsonExtract(obj, prop)} IS NULL` + }, + + set(array: string): string { + return `(SELECT value FROM json_each(${array}))` + }, + + epoch(): string { + return '00-00-00 00:00:00' + }, +} + +export default function make({ DATABASE }: Pick): Database { + const db = DATABASE as any + db.qb = qb + + return db as Database } diff --git a/backend/src/database/index.ts b/backend/src/database/index.ts index 01ab295..4d92787 100644 --- a/backend/src/database/index.ts +++ b/backend/src/database/index.ts @@ -13,6 +13,7 @@ export interface Database { dump(): Promise batch(statements: PreparedStatement[]): Promise[]> exec(query: string): Promise> + qb: QueryBuilder } export interface PreparedStatement { @@ -23,6 +24,13 @@ export interface PreparedStatement { raw(): Promise } +export interface QueryBuilder { + jsonExtract(obj: string, prop: string): string + jsonExtractIsNull(obj: string, prop: string): string + set(array: string): string + epoch(): string +} + export async function getDatabase(env: Pick): Promise { return d1(env) } diff --git a/backend/src/mastodon/timeline.ts b/backend/src/mastodon/timeline.ts index 57c945b..b57a283 100644 --- a/backend/src/mastodon/timeline.ts +++ b/backend/src/mastodon/timeline.ts @@ -16,7 +16,7 @@ export async function getHomeTimeline(domain: string, db: Database, actor: Actor ` SELECT actor_following.target_actor_id as id, - json_extract(actors.properties, '$.followers') as actorFollowersURL + ${db.qb.jsonExtract('actors.properties', 'followers')} as actorFollowersURL FROM actor_following INNER JOIN actors ON actors.id = actor_following.target_actor_id WHERE actor_id=? AND state='accepted' @@ -60,9 +60,9 @@ INNER JOIN objects ON objects.id = outbox_objects.object_id INNER JOIN actors ON actors.id = outbox_objects.actor_id WHERE objects.type = 'Note' - AND outbox_objects.actor_id IN (SELECT value FROM json_each(?2)) - AND json_extract(objects.properties, '$.inReplyTo') IS NULL - AND (outbox_objects.target = '${PUBLIC_GROUP}' OR outbox_objects.target IN (SELECT value FROM json_each(?3))) + AND outbox_objects.actor_id IN ${db.qb.set('?2')} + AND ${db.qb.jsonExtractIsNull('objects.properties', 'inReplyTo')} + AND (outbox_objects.target = '${PUBLIC_GROUP}' OR outbox_objects.target IN ${db.qb.set('?3')}) GROUP BY objects.id ORDER by outbox_objects.published_date DESC LIMIT ?4 @@ -101,7 +101,7 @@ export enum LocalPreference { function localPreferenceQuery(preference: LocalPreference): string { switch (preference) { case LocalPreference.NotSet: - return '1' + return 'true' case LocalPreference.OnlyLocal: return 'objects.local = 1' case LocalPreference.OnlyRemote: @@ -136,7 +136,7 @@ INNER JOIN actors ON actors.id=outbox_objects.actor_id LEFT JOIN note_hashtags ON objects.id=note_hashtags.object_id WHERE objects.type='Note' AND ${localPreferenceQuery(localPreference)} - AND json_extract(objects.properties, '$.inReplyTo') IS NULL + AND ${db.qb.jsonExtractIsNull('objects.properties', 'inReplyTo')} AND outbox_objects.target = '${PUBLIC_GROUP}' ${hashtagFilter} GROUP BY objects.id diff --git a/backend/test/mastodon/apps.spec.ts b/backend/test/mastodon/apps.spec.ts index b80edc7..998a2fa 100644 --- a/backend/test/mastodon/apps.spec.ts +++ b/backend/test/mastodon/apps.spec.ts @@ -76,12 +76,14 @@ describe('Mastodon APIs', () => { }) test('GET /apps is bad request', async () => { + const db = await makeDB() const vapidKeys = await generateVAPIDKeys() const request = new Request('https://example.com') const ctx: any = { next: () => new Response(), data: null, env: { + DATABASE: db, VAPID_JWK: JSON.stringify(vapidKeys), }, request, diff --git a/backend/test/mastodon/statuses.spec.ts b/backend/test/mastodon/statuses.spec.ts index d81c05f..3e834dd 100644 --- a/backend/test/mastodon/statuses.spec.ts +++ b/backend/test/mastodon/statuses.spec.ts @@ -81,7 +81,7 @@ describe('Mastodon APIs', () => { .prepare( ` SELECT - json_extract(properties, '$.content') as content, + ${db.qb.jsonExtract('properties', 'content')} as content, original_actor_id, original_object_id FROM objects @@ -758,7 +758,7 @@ describe('Mastodon APIs', () => { const row = await db .prepare( ` - SELECT json_extract(properties, '$.inReplyTo') as inReplyTo + SELECT ${db.qb.jsonExtract('properties', 'inReplyTo')} as inReplyTo FROM objects WHERE mastodon_id=? ` diff --git a/backend/test/utils.ts b/backend/test/utils.ts index f0b1471..7220ca7 100644 --- a/backend/test/utils.ts +++ b/backend/test/utils.ts @@ -9,6 +9,7 @@ import * as path from 'path' import { BetaDatabase } from '@miniflare/d1' import * as SQLiteDatabase from 'better-sqlite3' import { type Database } from 'wildebeest/backend/src/database' +import d1 from 'wildebeest/backend/src/database/d1' export function isUrlValid(s: string) { let url @@ -32,7 +33,8 @@ export async function makeDB(): Promise { db.exec(content) } - return db2 as unknown as Database + const env = { DATABASE: db2 } as any + return d1(env) } export function assertCORS(response: Response) { diff --git a/functions/api/v1/accounts/[id]/statuses.ts b/functions/api/v1/accounts/[id]/statuses.ts index ae44cfe..2ef1de2 100644 --- a/functions/api/v1/accounts/[id]/statuses.ts +++ b/functions/api/v1/accounts/[id]/statuses.ts @@ -140,7 +140,7 @@ FROM outbox_objects INNER JOIN objects ON objects.id=outbox_objects.object_id INNER JOIN actors ON actors.id=outbox_objects.actor_id WHERE objects.type='Note' - ${withReplies ? '' : "AND json_extract(objects.properties, '$.inReplyTo') IS NULL"} + ${withReplies ? '' : 'AND ' + db.qb.jsonExtractIsNull('objects.properties', 'inReplyTo')} AND outbox_objects.target = '${PUBLIC_GROUP}' AND outbox_objects.actor_id = ?1 AND outbox_objects.cdate > ?2 @@ -161,7 +161,7 @@ LIMIT ?3 OFFSET ?4 return new Response(JSON.stringify(out), { headers }) } - let afterCdate = '00-00-00 00:00:00' + let afterCdate = db.qb.epoch() if (url.searchParams.has('max_id')) { // Client asked to retrieve statuses after the max_id // As opposed to Mastodon we don't use incremental ID but UUID, we need From a606e760938ae9ed3b61dba46e32fc45754037ea Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 28 Feb 2023 09:39:41 +0000 Subject: [PATCH 16/62] handle hashtags in statuses - by parsing them in enrichStatus - implementing a tag page to show their timeline resolves #345 --- backend/src/mastodon/microformats.ts | 34 ++++++++- backend/test/mastodon.spec.ts | 45 ++++++++++++ .../components/StickyHeader/StickyHeader.tsx | 55 ++++++++------- frontend/src/dummyData/statuses.ts | 10 ++- .../routes/(frontend)/tags/[tag]/index.tsx | 69 +++++++++++++++++++ functions/api/v1/timelines/tag/[tag].ts | 17 +++-- ui-e2e-tests/seo.spec.ts | 11 +++ 7 files changed, 209 insertions(+), 32 deletions(-) create mode 100644 frontend/src/routes/(frontend)/tags/[tag]/index.tsx diff --git a/backend/src/mastodon/microformats.ts b/backend/src/mastodon/microformats.ts index c724958..72a4ee9 100644 --- a/backend/src/mastodon/microformats.ts +++ b/backend/src/mastodon/microformats.ts @@ -13,14 +13,24 @@ function tag(name: string, content: string, attrs: Record = {}): const linkRegex = /(^|\s|\b)(https?:\/\/[-\w@:%._+~#=]{2,256}\.[a-z]{2,6}\b(?:[-\w@:%_+.~#?&/=]*))(\b|\s|$)/g const mentionedEmailRegex = /(^|\s|\b|\W)@(\w+(?:[.-]?\w+)+@\w+(?:[.-]?\w+)+(?:\.\w{2,63})+)(\b|\s|$)/g +const tagRegex = /(^|\s|\b|\W)#(\w{2,63})(\b|\s|$)/g -/// Transform a text status into a HTML status; enriching it with links / mentions. +// Transform a text status into a HTML status; enriching it with links / mentions. export function enrichStatus(status: string, mentions: Array): string { - const enrichedStatus = status + const anchorsPlaceholdersMap = new Map() + + const getLinkAnchorPlaceholder = (link: string) => { + const anchor = getLinkAnchor(link) + const placeholder = `%%%___-LINK-PLACEHOLDER-${crypto.randomUUID()}-__%%%` + anchorsPlaceholdersMap.set(placeholder, anchor) + return placeholder + } + + let enrichedStatus = status .replace( linkRegex, (_, matchPrefix: string, link: string, matchSuffix: string) => - `${matchPrefix}${getLinkAnchor(link)}${matchSuffix}` + `${matchPrefix}${getLinkAnchorPlaceholder(link)}${matchSuffix}` ) .replace(mentionedEmailRegex, (_, matchPrefix: string, email: string, matchSuffix: string) => { // ensure that the match is part of the mentions array @@ -33,6 +43,15 @@ export function enrichStatus(status: string, mentions: Array): string { // otherwise the match isn't valid and we don't add HTML return `${matchPrefix}${email}${matchSuffix}` }) + .replace( + tagRegex, + (_, matchPrefix: string, tag: string, matchSuffix: string) => + `${matchPrefix}${/^\d+$/.test(tag) ? `#${tag}` : getTagAnchor(tag)}${matchSuffix}` + ) + + for (const [placeholder, anchor] of anchorsPlaceholdersMap.entries()) { + enrichedStatus = enrichedStatus.replace(placeholder, anchor) + } return tag('p', enrichedStatus) } @@ -60,3 +79,12 @@ function getLinkAnchor(link: string) { return link } } + +function getTagAnchor(hashTag: string) { + try { + return tag('a', `#${hashTag}`, { href: `/tags/${hashTag.replace(/^#/, '')}`, class: 'status-link hashtag' }) + } catch (err: unknown) { + console.warn('failed to parse link', err) + return tag + } +} diff --git a/backend/test/mastodon.spec.ts b/backend/test/mastodon.spec.ts index 3773a4b..c5ca0ea 100644 --- a/backend/test/mastodon.spec.ts +++ b/backend/test/mastodon.spec.ts @@ -198,6 +198,51 @@ describe('Mastodon APIs', () => { assert.equal(enrichStatus(`@!@£${link}!!!`, []), `

@!@£${urlDisplayText}!!!

`) }) }) + + test('convert tags to HTML', async () => { + const tagsToTest = [ + { + tag: '#test', + expectedTagAnchor: '#test', + }, + { + tag: '#123_joke_123', + expectedTagAnchor: '#123_joke_123', + }, + { + tag: '#_123', + expectedTagAnchor: '#_123', + }, + { + tag: '#example:', + expectedTagAnchor: '#example:', + }, + { + tag: '#tagA#tagB', + expectedTagAnchor: + '#tagA#tagB', + }, + ] + + for (let i = 0, len = tagsToTest.length; i < len; i++) { + const { tag, expectedTagAnchor } = tagsToTest[i] + + assert.equal(enrichStatus(`hey ${tag} hi`, []), `

hey ${expectedTagAnchor} hi

`) + assert.equal(enrichStatus(`${tag} hi`, []), `

${expectedTagAnchor} hi

`) + assert.equal(enrichStatus(`${tag}\n\thein`, []), `

${expectedTagAnchor}\n\thein

`) + assert.equal(enrichStatus(`hey ${tag}`, []), `

hey ${expectedTagAnchor}

`) + assert.equal(enrichStatus(`${tag}`, []), `

${expectedTagAnchor}

`) + assert.equal(enrichStatus(`@!@£${tag}!!!`, []), `

@!@£${expectedTagAnchor}!!!

`) + } + }) + + test('ignore invalid tags', () => { + assert.equal(enrichStatus('tags cannot be empty like: #', []), `

tags cannot be empty like: #

`) + assert.equal( + enrichStatus('tags cannot contain only numbers like: #123', []), + `

tags cannot contain only numbers like: #123

` + ) + }) }) describe('Follow', () => { diff --git a/frontend/src/components/StickyHeader/StickyHeader.tsx b/frontend/src/components/StickyHeader/StickyHeader.tsx index 8266ffc..d0a3643 100644 --- a/frontend/src/components/StickyHeader/StickyHeader.tsx +++ b/frontend/src/components/StickyHeader/StickyHeader.tsx @@ -1,30 +1,37 @@ import { $, component$, Slot } from '@builder.io/qwik' import { useNavigate } from '@builder.io/qwik-city' -export default component$<{ withBackButton?: boolean }>(({ withBackButton }) => { - const nav = useNavigate() +export default component$<{ withBackButton?: boolean; backButtonPlacement?: 'start' | 'end' }>( + ({ withBackButton, backButtonPlacement = 'start' }) => { + const nav = useNavigate() - const goBack = $(() => { - if (window.history.length > 1) { - window.history.back() - } else { - nav('/explore') - } - }) + const goBack = $(() => { + if (window.history.length > 1) { + window.history.back() + } else { + nav('/explore') + } + }) - return ( -
-
- {!!withBackButton && ( -
- -
- )} - + const backButton = !withBackButton ? ( + // eslint-disable-next-line qwik/single-jsx-root + <> + ) : ( +
+
-
- ) -}) + ) + return ( +
+
+ {backButtonPlacement === 'start' && backButton} + + {backButtonPlacement === 'end' &&
{backButton}
} +
+
+ ) + } +) diff --git a/frontend/src/dummyData/statuses.ts b/frontend/src/dummyData/statuses.ts index 5125d94..a505309 100644 --- a/frontend/src/dummyData/statuses.ts +++ b/frontend/src/dummyData/statuses.ts @@ -31,7 +31,15 @@ const mastodonRawStatuses: MastodonStatus[] = [ content: 'A very simple update: all good!', account: ben, }), - generateDummyStatus({ content: '

Hi! My name is Rafael! 👋

', account: rafael, spoiler_text: 'who am I?' }), + generateDummyStatus({ + content: '

Hi! My name is Rafael! 👋

', + account: rafael, + spoiler_text: 'who am I?', + }), + generateDummyStatus({ + content: '

Hi! I made a funny! 🤭 #joke

', + account: george, + }), generateDummyStatus({ content: "

I'm Rafael and I am a web designer!

💪💪

", account: rafael, diff --git a/frontend/src/routes/(frontend)/tags/[tag]/index.tsx b/frontend/src/routes/(frontend)/tags/[tag]/index.tsx new file mode 100644 index 0000000..9766f46 --- /dev/null +++ b/frontend/src/routes/(frontend)/tags/[tag]/index.tsx @@ -0,0 +1,69 @@ +import { $, component$ } from '@builder.io/qwik' +import { DocumentHead, loader$ } from '@builder.io/qwik-city' +import { getDatabase } from 'wildebeest/backend/src/database' +import { getDomain } from 'wildebeest/backend/src/utils/getDomain' +import { handleRequest } from 'wildebeest/functions/api/v1/timelines/tag/[tag]' +import { StatusesPanel } from '~/components/StatusesPanel/StatusesPanel' +import StickyHeader from '~/components/StickyHeader/StickyHeader' +import { MastodonStatus } from '~/types' +import { getDocumentHead } from '~/utils/getDocumentHead' + +export const loader = loader$, { DATABASE: D1Database }>( + async ({ request, platform, params }) => { + const tag = params.tag + const response = await handleRequest(await getDatabase(platform), request, getDomain(request.url), tag) + const results = await response.text() + const statuses: MastodonStatus[] = JSON.parse(results) + return { tag, statuses } + } +) + +export default component$(() => { + const loaderData = loader() + + return ( + <> +
+ +

+ + {loaderData.value.tag} +

+
+ { + let statuses: MastodonStatus[] = [] + try { + const response = await fetch( + `/api/v1/timelines/tags/${loaderData.value.tag}/?offset=${numOfCurrentStatuses}` + ) + if (response.ok) { + const results = await response.text() + statuses = JSON.parse(results) + } + } catch { + /* empty */ + } + return statuses + })} + /> +
+ + ) +}) + +export const requestUrlLoader = loader$(async ({ request }) => request.url) + +export const head: DocumentHead = ({ resolveValue }) => { + const { tag } = resolveValue(loader) + const url = resolveValue(requestUrlLoader) + + return getDocumentHead({ + title: `#${tag} - Wildebeest`, + description: `#${tag} tag page - Wildebeest`, + og: { + url, + }, + }) +} diff --git a/functions/api/v1/timelines/tag/[tag].ts b/functions/api/v1/timelines/tag/[tag].ts index 9074fe4..7105dc4 100644 --- a/functions/api/v1/timelines/tag/[tag].ts +++ b/functions/api/v1/timelines/tag/[tag].ts @@ -3,6 +3,7 @@ import { cors } from 'wildebeest/backend/src/utils/cors' import type { ContextData } from 'wildebeest/backend/src/types/context' import * as timelines from 'wildebeest/backend/src/mastodon/timeline' import { type Database, getDatabase } from 'wildebeest/backend/src/database' +import { getDomain } from 'wildebeest/backend/src/utils/getDomain' const headers = { ...cors(), @@ -10,16 +11,24 @@ const headers = { } export const onRequest: PagesFunction = async ({ request, env, params }) => { - const domain = new URL(request.url).hostname - return handleRequest(await getDatabase(env), request, domain, params.tag as string) + const url = new URL(request.url) + const { searchParams } = url + const offset = Number.parseInt(searchParams.get('offset') ?? '0') + return handleRequest(await getDatabase(env), request, getDomain(url), params.tag as string, offset) } -export async function handleRequest(db: Database, request: Request, domain: string, tag: string): Promise { +export async function handleRequest( + db: Database, + request: Request, + domain: string, + tag: string, + offset = 0 +): Promise { const url = new URL(request.url) if (url.searchParams.has('max_id')) { return new Response(JSON.stringify([]), { headers }) } - const timeline = await timelines.getPublicTimeline(domain, db, timelines.LocalPreference.NotSet, 0, tag) + const timeline = await timelines.getPublicTimeline(domain, db, timelines.LocalPreference.NotSet, offset, tag) return new Response(JSON.stringify(timeline), { headers }) } diff --git a/ui-e2e-tests/seo.spec.ts b/ui-e2e-tests/seo.spec.ts index 92cdafc..0c421d0 100644 --- a/ui-e2e-tests/seo.spec.ts +++ b/ui-e2e-tests/seo.spec.ts @@ -71,6 +71,17 @@ test.describe('Presence of appropriate SEO metadata across the application', () }) }) + test('in tag page', async ({ page }) => { + await page.goto('http://127.0.0.1:8788/tags/my-tag') + await checkPageSeoData(page, { + title: '#my-tag - Wildebeest', + description: '#my-tag tag page - Wildebeest', + ogType: 'website', + ogUrl: 'http://127.0.0.1:8788/tags/my-tag', + ogImage: 'https://imagedelivery.net/NkfPDviynOyTAOI79ar_GQ/b24caf12-5230-48c4-0bf7-2f40063bd400/thumbnail', + }) + }) + // To unskip when we enable the about page test.skip('in about page', async ({ page }) => { await page.goto('http://127.0.0.1:8788/about') From 1b830d8df233d898a112ebf7c2cd663a1fc491e3 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 1 Mar 2023 09:20:59 +0000 Subject: [PATCH 17/62] query builder insertOrIgnore --- backend/src/activitypub/peers.ts | 7 +++---- backend/src/database/d1.ts | 4 ++++ backend/src/database/index.ts | 1 + backend/src/mastodon/follow.ts | 20 +++++++++++--------- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/backend/src/activitypub/peers.ts b/backend/src/activitypub/peers.ts index 3f05220..bd2a6b0 100644 --- a/backend/src/activitypub/peers.ts +++ b/backend/src/activitypub/peers.ts @@ -9,10 +9,9 @@ export async function getPeers(db: Database): Promise> { } export async function addPeer(db: Database, domain: string): Promise { - const query = ` - INSERT OR IGNORE INTO peers (domain) - VALUES (?) - ` + const query = db.qb.insertOrIgnore(` + INTO peers (domain) VALUES (?) + `) const out = await db.prepare(query).bind(domain).run() if (!out.success) { diff --git a/backend/src/database/d1.ts b/backend/src/database/d1.ts index 41c7fe4..101b0da 100644 --- a/backend/src/database/d1.ts +++ b/backend/src/database/d1.ts @@ -17,6 +17,10 @@ const qb: QueryBuilder = { epoch(): string { return '00-00-00 00:00:00' }, + + insertOrIgnore(q: string): string { + return `INSERT OR IGNORE ${q}` + }, } export default function make({ DATABASE }: Pick): Database { diff --git a/backend/src/database/index.ts b/backend/src/database/index.ts index 4d92787..f516827 100644 --- a/backend/src/database/index.ts +++ b/backend/src/database/index.ts @@ -29,6 +29,7 @@ export interface QueryBuilder { jsonExtractIsNull(obj: string, prop: string): string set(array: string): string epoch(): string + insertOrIgnore(q: string): string } export async function getDatabase(env: Pick): Promise { diff --git a/backend/src/mastodon/follow.ts b/backend/src/mastodon/follow.ts index 49f0552..db563a3 100644 --- a/backend/src/mastodon/follow.ts +++ b/backend/src/mastodon/follow.ts @@ -10,11 +10,12 @@ const STATE_ACCEPTED = 'accepted' // During a migration we move the followers from the old Actor to the new export async function moveFollowers(db: Database, actor: Actor, followers: Array): Promise { const batch = [] - const stmt = db.prepare(` - INSERT OR IGNORE + const stmt = db.prepare( + db.qb.insertOrIgnore(` INTO actor_following (id, actor_id, target_actor_id, target_actor_acct, state) - VALUES (?1, ?2, ?3, ?4, 'accepted'); + VALUES (?1, ?2, ?3, ?4, 'accepted') `) + ) const actorId = actor.id.toString() const actorAcc = urlToHandle(actor.id) @@ -32,11 +33,12 @@ export async function moveFollowers(db: Database, actor: Actor, followers: Array export async function moveFollowing(db: Database, actor: Actor, followingActors: Array): Promise { const batch = [] - const stmt = db.prepare(` - INSERT OR IGNORE + const stmt = db.prepare( + db.qb.insertOrIgnore(` INTO actor_following (id, actor_id, target_actor_id, target_actor_acct, state) - VALUES (?1, ?2, ?3, ?4, 'accepted'); + VALUES (?1, ?2, ?3, ?4, 'accepted') `) + ) const actorId = actor.id.toString() @@ -56,10 +58,10 @@ export async function moveFollowing(db: Database, actor: Actor, followingActors: export async function addFollowing(db: Database, actor: Actor, target: Actor, targetAcct: string): Promise { const id = crypto.randomUUID() - const query = ` - INSERT OR IGNORE INTO actor_following (id, actor_id, target_actor_id, state, target_actor_acct) + const query = db.qb.insertOrIgnore(` + INTO actor_following (id, actor_id, target_actor_id, state, target_actor_acct) VALUES (?, ?, ?, ?, ?) - ` + `) const out = await db .prepare(query) From c3fb612d21e84a8ca0e2d3690bed4135a46b3b2a Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 1 Mar 2023 09:30:35 +0000 Subject: [PATCH 18/62] add neon client --- .github/workflows/deploy.yml | 1 + backend/src/database/index.ts | 7 ++- backend/src/database/neon.ts | 111 ++++++++++++++++++++++++++++++++++ backend/src/types/env.ts | 2 + package.json | 1 + yarn.lock | 5 ++ 6 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 backend/src/database/neon.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 24d4cb7..e3c45ed 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -273,6 +273,7 @@ jobs: echo -e "DOMAIN=\"${{ vars.CF_DEPLOY_DOMAIN }}\"\n" >> consumer/wrangler.toml echo -e "ADMIN_EMAIL=\"${{ vars.ADMIN_EMAIL }}\"\n" >> consumer/wrangler.toml + yarn yarn --cwd consumer/ echo "******" command: publish --config consumer/wrangler.toml diff --git a/backend/src/database/index.ts b/backend/src/database/index.ts index f516827..72cfe86 100644 --- a/backend/src/database/index.ts +++ b/backend/src/database/index.ts @@ -1,5 +1,6 @@ import type { Env } from 'wildebeest/backend/src/types/env' import d1 from './d1' +import neon from './neon' export interface Result { results?: T[] @@ -32,6 +33,10 @@ export interface QueryBuilder { insertOrIgnore(q: string): string } -export async function getDatabase(env: Pick): Promise { +export async function getDatabase(env: Pick): Promise { + if (env.NEON_DATABASE_URL !== undefined) { + return neon(env) + } + return d1(env) } diff --git a/backend/src/database/neon.ts b/backend/src/database/neon.ts new file mode 100644 index 0000000..b2bec7d --- /dev/null +++ b/backend/src/database/neon.ts @@ -0,0 +1,111 @@ +import * as neon from '@neondatabase/serverless' +import type { Database, Result, QueryBuilder } from 'wildebeest/backend/src/database' +import type { Env } from 'wildebeest/backend/src/types/env' + +function sqliteToPsql(query: string): string { + let c = 0 + return query.replaceAll(/\?([0-9])?/g, (match: string, p1: string) => { + c += 1 + return `$${p1 || c}` + }) +} + +const qb: QueryBuilder = { + jsonExtract(obj: string, prop: string): string { + return `json_extract_path(${obj}::json, '${prop}')::text` + }, + + jsonExtractIsNull(obj: string, prop: string): string { + return `${qb.jsonExtract(obj, prop)} = 'null'` + }, + + set(array: string): string { + return `(SELECT value::text FROM json_array_elements_text(${array}))` + }, + + epoch(): string { + return 'epoch' + }, + + insertOrIgnore(q: string): string { + return `INSERT ${q} ON CONFLICT DO NOTHING` + }, +} + +export default async function make(env: Pick): Promise { + const client = new neon.Client(env.NEON_DATABASE_URL) + await client.connect() + + return { + qb, + + prepare(query: string) { + return new PreparedStatement(env, query, [], client) + }, + + dump() { + throw new Error('not implemented') + }, + + async batch(statements: PreparedStatement[]): Promise[]> { + throw new Error('not implemented') + console.log(statements) + }, + + async exec(query: string): Promise> { + throw new Error('not implemented') + console.log(query) + }, + } +} + +export class PreparedStatement { + private env: Pick + private client: neon.Client + private query: string + private values: any[] + + constructor(env: Pick, query: string, values: any[], client: neon.Client) { + this.env = env + this.query = query + this.values = values + this.client = client + } + + bind(...values: any[]): PreparedStatement { + return new PreparedStatement(this.env, this.query, [...this.values, ...values], this.client) + } + + async first(colName?: string): Promise { + if (colName) { + throw new Error('not implemented') + } + const query = sqliteToPsql(this.query) + + const results = await this.client.query(query, this.values) + if (results.rows.length !== 1) { + throw new Error(`expected a single row, returned ${results.rows.length} row(s)`) + } + + return results.rows[0] as T + } + + async run(): Promise> { + return this.all() + } + + async all(): Promise> { + const query = sqliteToPsql(this.query) + const results = await this.client.query(query, this.values) + + return { + results: results.rows as T[], + success: true, + meta: {}, + } + } + + async raw(): Promise { + throw new Error('not implemented') + } +} diff --git a/backend/src/types/env.ts b/backend/src/types/env.ts index d849c5e..1862a70 100644 --- a/backend/src/types/env.ts +++ b/backend/src/types/env.ts @@ -25,4 +25,6 @@ export interface Env { SENTRY_DSN: string SENTRY_ACCESS_CLIENT_ID: string SENTRY_ACCESS_CLIENT_SECRET: string + + NEON_DATABASE_URL?: string } diff --git a/package.json b/package.json index 82315a8..91501f3 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "deploy": "yarn build && yarn database:migrate && yarn pages publish frontend/dist --project-name=wildebeest" }, "dependencies": { + "@neondatabase/serverless": "^0.2.5", "@types/cookie": "^0.5.1", "cookie": "^0.5.0", "http-message-signatures": "^0.1.2", diff --git a/yarn.lock b/yarn.lock index 0e7addd..773bce4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -807,6 +807,11 @@ undici "5.9.1" ws "^8.2.2" +"@neondatabase/serverless@^0.2.5": + version "0.2.5" + resolved "https://registry.yarnpkg.com/@neondatabase/serverless/-/serverless-0.2.5.tgz#78bd4a905f50d087d06f16c02711fdd05b2a851d" + integrity sha512-Qu/nNZftfoqw4ojVCXU/EgYlfII3mzLm82iXNOUljFumPhoZ/Wp8NJG5DgSAKCWC0zwTyJsojdPLQDj/UPs2vg== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz" From c4d0c69b22b5dcd2ccaa49d1e8e325cdda952621 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 1 Mar 2023 09:37:23 +0000 Subject: [PATCH 19/62] query builder psqlOnly --- backend/src/database/d1.ts | 4 ++++ backend/src/database/index.ts | 1 + backend/src/database/neon.ts | 4 ++++ backend/src/mastodon/timeline.ts | 6 ++++-- functions/api/v1/accounts/[id]/statuses.ts | 2 +- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/src/database/d1.ts b/backend/src/database/d1.ts index 101b0da..a512d97 100644 --- a/backend/src/database/d1.ts +++ b/backend/src/database/d1.ts @@ -21,6 +21,10 @@ const qb: QueryBuilder = { insertOrIgnore(q: string): string { return `INSERT OR IGNORE ${q}` }, + + psqlOnly(): string { + return '' + }, } export default function make({ DATABASE }: Pick): Database { diff --git a/backend/src/database/index.ts b/backend/src/database/index.ts index 72cfe86..0e854fc 100644 --- a/backend/src/database/index.ts +++ b/backend/src/database/index.ts @@ -31,6 +31,7 @@ export interface QueryBuilder { set(array: string): string epoch(): string insertOrIgnore(q: string): string + psqlOnly(raw: string): string } export async function getDatabase(env: Pick): Promise { diff --git a/backend/src/database/neon.ts b/backend/src/database/neon.ts index b2bec7d..c4c1f6d 100644 --- a/backend/src/database/neon.ts +++ b/backend/src/database/neon.ts @@ -30,6 +30,10 @@ const qb: QueryBuilder = { insertOrIgnore(q: string): string { return `INSERT ${q} ON CONFLICT DO NOTHING` }, + + psqlOnly(q: string): string { + return q + }, } export default async function make(env: Pick): Promise { diff --git a/backend/src/mastodon/timeline.ts b/backend/src/mastodon/timeline.ts index b57a283..7627897 100644 --- a/backend/src/mastodon/timeline.ts +++ b/backend/src/mastodon/timeline.ts @@ -63,7 +63,7 @@ WHERE AND outbox_objects.actor_id IN ${db.qb.set('?2')} AND ${db.qb.jsonExtractIsNull('objects.properties', 'inReplyTo')} AND (outbox_objects.target = '${PUBLIC_GROUP}' OR outbox_objects.target IN ${db.qb.set('?3')}) -GROUP BY objects.id +GROUP BY objects.id ${db.qb.psqlOnly(', actors.id, outbox_objects.actor_id, outbox_objects.published_date')} ORDER by outbox_objects.published_date DESC LIMIT ?4 ` @@ -139,7 +139,9 @@ WHERE objects.type='Note' AND ${db.qb.jsonExtractIsNull('objects.properties', 'inReplyTo')} AND outbox_objects.target = '${PUBLIC_GROUP}' ${hashtagFilter} -GROUP BY objects.id +GROUP BY objects.id ${db.qb.psqlOnly( + ', actors.id, actors.cdate, actors.properties, outbox_objects.actor_id, outbox_objects.published_date' + )} ORDER by outbox_objects.published_date DESC LIMIT ?1 OFFSET ?2 ` diff --git a/functions/api/v1/accounts/[id]/statuses.ts b/functions/api/v1/accounts/[id]/statuses.ts index 2ef1de2..04bc3f3 100644 --- a/functions/api/v1/accounts/[id]/statuses.ts +++ b/functions/api/v1/accounts/[id]/statuses.ts @@ -143,7 +143,7 @@ WHERE objects.type='Note' ${withReplies ? '' : 'AND ' + db.qb.jsonExtractIsNull('objects.properties', 'inReplyTo')} AND outbox_objects.target = '${PUBLIC_GROUP}' AND outbox_objects.actor_id = ?1 - AND outbox_objects.cdate > ?2 + AND outbox_objects.cdate > ?2${db.qb.psqlOnly('::timestamp')} ORDER by outbox_objects.published_date DESC LIMIT ?3 OFFSET ?4 ` From 613ada3f8e72483e29e7723758726af197459dc8 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 1 Mar 2023 11:46:09 +0000 Subject: [PATCH 20/62] consumer: add missing NEON_DATABASE_URL --- consumer/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/consumer/src/index.ts b/consumer/src/index.ts index fa6d72e..ada414d 100644 --- a/consumer/src/index.ts +++ b/consumer/src/index.ts @@ -15,6 +15,8 @@ export type Env = { SENTRY_DSN: string SENTRY_ACCESS_CLIENT_ID: string SENTRY_ACCESS_CLIENT_SECRET: string + + NEON_DATABASE_URL?: string } export default { From 2c947733a27ca9ad4d269e3a0626c4aa391f44c6 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 1 Mar 2023 11:46:18 +0000 Subject: [PATCH 21/62] neon: implement batch query --- backend/src/database/neon.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/backend/src/database/neon.ts b/backend/src/database/neon.ts index c4c1f6d..802c270 100644 --- a/backend/src/database/neon.ts +++ b/backend/src/database/neon.ts @@ -52,8 +52,20 @@ export default async function make(env: Pick): Promise }, async batch(statements: PreparedStatement[]): Promise[]> { - throw new Error('not implemented') - console.log(statements) + const results = [] + + for (let i = 0, len = statements.length; i < len; i++) { + const query = sqliteToPsql(statements[i].query) + const result = await client.query(query, statements[i].values) + + results.push({ + results: result.rows as T[], + success: true, + meta: {}, + }) + } + + return results }, async exec(query: string): Promise> { @@ -66,8 +78,8 @@ export default async function make(env: Pick): Promise export class PreparedStatement { private env: Pick private client: neon.Client - private query: string - private values: any[] + public query: string + public values: any[] constructor(env: Pick, query: string, values: any[], client: neon.Client) { this.env = env From ba4daecf17ec3ccbff49cab30288f8e3dcf3de48 Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 1 Mar 2023 12:24:25 +0000 Subject: [PATCH 22/62] fix byte array for neon --- backend/src/activitypub/actors/index.ts | 3 ++- backend/src/database/d1.ts | 1 + backend/src/database/index.ts | 1 + backend/src/database/neon.ts | 1 + backend/src/mastodon/account.ts | 9 ++++++++- consumer/wrangler.toml | 1 + wrangler.toml | 1 + 7 files changed, 15 insertions(+), 2 deletions(-) diff --git a/backend/src/activitypub/actors/index.ts b/backend/src/activitypub/actors/index.ts index 7156827..cd89ef5 100644 --- a/backend/src/activitypub/actors/index.ts +++ b/backend/src/activitypub/actors/index.ts @@ -3,6 +3,7 @@ import { generateUserKey } from 'wildebeest/backend/src/utils/key-ops' import { type APObject, sanitizeContent, getTextContent } from '../objects' import { addPeer } from 'wildebeest/backend/src/activitypub/peers' import { type Database } from 'wildebeest/backend/src/database' +import { Buffer } from 'buffer' const PERSON = 'Person' const isTesting = typeof jest !== 'undefined' @@ -158,7 +159,7 @@ export async function createPerson( // Since D1 and better-sqlite3 behaviors don't exactly match, presumable // because Buffer support is different in Node/Worker. We have to transform // the values depending on the platform. - if (isTesting) { + if (isTesting || db.client === 'neon') { privkey = Buffer.from(userKeyPair.wrappedPrivKey) salt = Buffer.from(userKeyPair.salt) } else { diff --git a/backend/src/database/d1.ts b/backend/src/database/d1.ts index a512d97..ba6af22 100644 --- a/backend/src/database/d1.ts +++ b/backend/src/database/d1.ts @@ -30,6 +30,7 @@ const qb: QueryBuilder = { export default function make({ DATABASE }: Pick): Database { const db = DATABASE as any db.qb = qb + db.client = 'd1' return db as Database } diff --git a/backend/src/database/index.ts b/backend/src/database/index.ts index 0e854fc..edcc402 100644 --- a/backend/src/database/index.ts +++ b/backend/src/database/index.ts @@ -15,6 +15,7 @@ export interface Database { batch(statements: PreparedStatement[]): Promise[]> exec(query: string): Promise> qb: QueryBuilder + client: string } export interface PreparedStatement { diff --git a/backend/src/database/neon.ts b/backend/src/database/neon.ts index 802c270..235ded5 100644 --- a/backend/src/database/neon.ts +++ b/backend/src/database/neon.ts @@ -41,6 +41,7 @@ export default async function make(env: Pick): Promise await client.connect() return { + client: 'neon', qb, prepare(query: string) { diff --git a/backend/src/mastodon/account.ts b/backend/src/mastodon/account.ts index 7c81612..170b675 100644 --- a/backend/src/mastodon/account.ts +++ b/backend/src/mastodon/account.ts @@ -89,5 +89,12 @@ SELECT export async function getSigningKey(instanceKey: string, db: Database, actor: Actor): Promise { const stmt = db.prepare('SELECT privkey, privkey_salt FROM actors WHERE id=?').bind(actor.id.toString()) const { privkey, privkey_salt } = (await stmt.first()) as any - return unwrapPrivateKey(instanceKey, new Uint8Array(privkey), new Uint8Array(privkey_salt)) + + if (privkey.buffer && privkey_salt.buffer) { + // neon.tech + return unwrapPrivateKey(instanceKey, new Uint8Array(privkey.buffer), new Uint8Array(privkey_salt.buffer)) + } else { + // D1 + return unwrapPrivateKey(instanceKey, new Uint8Array(privkey), new Uint8Array(privkey_salt)) + } } diff --git a/consumer/wrangler.toml b/consumer/wrangler.toml index 7a861f3..8d2045f 100644 --- a/consumer/wrangler.toml +++ b/consumer/wrangler.toml @@ -1,3 +1,4 @@ compatibility_date = "2023-01-09" main = "./src/index.ts" usage_model = "unbound" +node_compat = true diff --git a/wrangler.toml b/wrangler.toml index 6dd839d..ea7d914 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -3,6 +3,7 @@ name = "wildebeest" main = "src/index.ts" compatibility_date = "2022-12-05" +node_compat = true # Specify your account id here so that all commands run agains the correct account. # account_id = "" From 781b46050f5581db9aa38f617c609769d1ed049c Mon Sep 17 00:00:00 2001 From: kelvin Date: Thu, 2 Mar 2023 00:42:08 +0900 Subject: [PATCH 23/62] allow website to be undefined --- backend/src/mastodon/client.ts | 6 +++--- backend/test/mastodon/apps.spec.ts | 27 +++++++++++++++++++++++++++ backend/test/utils.ts | 2 +- functions/api/v1/apps.ts | 13 +++++++++++-- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/backend/src/mastodon/client.ts b/backend/src/mastodon/client.ts index 2c2df3d..db8472b 100644 --- a/backend/src/mastodon/client.ts +++ b/backend/src/mastodon/client.ts @@ -6,16 +6,16 @@ export interface Client { secret: string name: string redirect_uris: string - website: string scopes: string + website?: string } export async function createClient( db: Database, name: string, redirect_uris: string, - website: string, - scopes: string + scopes: string, + website?: string ): Promise { const id = crypto.randomUUID() diff --git a/backend/test/mastodon/apps.spec.ts b/backend/test/mastodon/apps.spec.ts index 998a2fa..17896f5 100644 --- a/backend/test/mastodon/apps.spec.ts +++ b/backend/test/mastodon/apps.spec.ts @@ -36,6 +36,33 @@ describe('Mastodon APIs', () => { assert.deepEqual(rest, {}) }) + test('POST /apps registers client without website', async () => { + const db = await makeDB() + const vapidKeys = await generateVAPIDKeys() + const request = new Request('https://example.com', { + method: 'POST', + body: '{"redirect_uris":"mastodon://example.com/oauth","client_name":"Example mastodon client","scopes":"read write follow push"}', + headers: { + 'content-type': 'application/json', + }, + }) + + const res = await apps.handleRequest(db, request, vapidKeys) + assert.equal(res.status, 200) + assertCORS(res) + assertJSON(res) + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { name, redirect_uri, client_id, client_secret, vapid_key, id, ...rest } = await res.json< + Record + >() + + assert.equal(name, 'Example mastodon client') + assert.equal(redirect_uri, 'mastodon://example.com/oauth') + assert.equal(id, '20') + assert.deepEqual(rest, {}) + }) + test('POST /apps returns 422 for malformed requests', async () => { // client_name and redirect_uris are required according to https://docs.joinmastodon.org/methods/apps/#form-data-parameters const db = await makeDB() diff --git a/backend/test/utils.ts b/backend/test/utils.ts index 7220ca7..418607e 100644 --- a/backend/test/utils.ts +++ b/backend/test/utils.ts @@ -73,7 +73,7 @@ export async function createTestClient( redirectUri: string = 'https://localhost', scopes: string = 'read follow' ): Promise { - return createClient(db, 'test client', redirectUri, 'https://cloudflare.com', scopes) + return createClient(db, 'test client', redirectUri, scopes, 'https://cloudflare.com') } type TestQueue = Queue & { messages: Array } diff --git a/functions/api/v1/apps.ts b/functions/api/v1/apps.ts index 32d9f17..ef19120 100644 --- a/functions/api/v1/apps.ts +++ b/functions/api/v1/apps.ts @@ -11,7 +11,7 @@ import { type Database, getDatabase } from 'wildebeest/backend/src/database' type AppsPost = { redirect_uris: string - website: string + website?: string client_name: string scopes: string } @@ -42,9 +42,18 @@ export async function handleRequest(db: Database, request: Request, vapidKeys: J } catch { return errors.unprocessableEntity('redirect_uris must be a valid URI') } + } else if (body.website) { + if (body.website.length > 2000) { + return errors.unprocessableEntity('website cannot exceed 2000 characters') + } + try { + new URL('', body.website) + } catch { + return errors.unprocessableEntity('website is invalid URI') + } } - const client = await createClient(db, body.client_name, body.redirect_uris, body.website, body.scopes) + const client = await createClient(db, body.client_name, body.redirect_uris, body.scopes, body.website) const vapidKey = VAPIDPublicKey(vapidKeys) const res = { From 246edfc78944479da5db9b06d5ca204c9dfb6090 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 24 Feb 2023 15:53:10 +0000 Subject: [PATCH 24/62] add both FE and BE for server settings (including rules) --- backend/src/activitypub/actors/index.ts | 9 +- frontend/mock-db/init.ts | 19 ++- frontend/src/components/Settings/TextArea.tsx | 34 +++++ .../src/components/Settings/TextInput.tsx | 36 ++++++ .../routes/(admin)/oauth/authorize/index.tsx | 79 +++++------- .../settings/server-settings/about/index.tsx | 69 ++++++++++ .../server-settings/branding/index.tsx | 71 ++++++++++ .../settings/server-settings/index.tsx | 8 ++ .../settings/server-settings/layout.tsx | 74 +++++++++++ .../server-settings/rules/edit/[id]/index.tsx | 96 ++++++++++++++ .../settings/server-settings/rules/index.tsx | 122 ++++++++++++++++++ .../routes/(frontend)/[accountId]/layout.tsx | 3 +- .../src/routes/(frontend)/about/index.tsx | 108 +++++++++------- frontend/src/routes/(frontend)/layout.tsx | 5 +- frontend/src/utils/getJwtEmail.ts | 23 ++++ frontend/src/utils/isUserAdmin.ts | 17 +++ frontend/yarn.lock | 56 +------- functions/api/v1/instance/rules.ts | 18 +++ functions/api/wb/settings/server/admins.ts | 26 ++++ functions/api/wb/settings/server/rules.ts | 65 ++++++++++ functions/api/wb/settings/server/server.ts | 64 +++++++++ functions/first-login.ts | 21 ++- migrations/0008_add_server-settings.sql | 11 ++ package.json | 4 +- 24 files changed, 864 insertions(+), 174 deletions(-) create mode 100644 frontend/src/components/Settings/TextArea.tsx create mode 100644 frontend/src/components/Settings/TextInput.tsx create mode 100644 frontend/src/routes/(admin)/settings/server-settings/about/index.tsx create mode 100644 frontend/src/routes/(admin)/settings/server-settings/branding/index.tsx create mode 100644 frontend/src/routes/(admin)/settings/server-settings/index.tsx create mode 100644 frontend/src/routes/(admin)/settings/server-settings/layout.tsx create mode 100644 frontend/src/routes/(admin)/settings/server-settings/rules/edit/[id]/index.tsx create mode 100644 frontend/src/routes/(admin)/settings/server-settings/rules/index.tsx create mode 100644 frontend/src/utils/getJwtEmail.ts create mode 100644 frontend/src/utils/isUserAdmin.ts create mode 100644 functions/api/v1/instance/rules.ts create mode 100644 functions/api/wb/settings/server/admins.ts create mode 100644 functions/api/wb/settings/server/rules.ts create mode 100644 functions/api/wb/settings/server/server.ts create mode 100644 migrations/0008_add_server-settings.sql diff --git a/backend/src/activitypub/actors/index.ts b/backend/src/activitypub/actors/index.ts index 7156827..25f64b1 100644 --- a/backend/src/activitypub/actors/index.ts +++ b/backend/src/activitypub/actors/index.ts @@ -150,7 +150,8 @@ export async function createPerson( db: Database, userKEK: string, email: string, - properties: PersonProperties = {} + properties: PersonProperties = {}, + admin: boolean = false ): Promise { const userKeyPair = await generateUserKey(userKEK) @@ -198,12 +199,12 @@ export async function createPerson( const row = await db .prepare( ` - INSERT INTO actors(id, type, email, pubkey, privkey, privkey_salt, properties) - VALUES(?, ?, ?, ?, ?, ?, ?) + INSERT INTO actors(id, type, email, pubkey, privkey, privkey_salt, properties, is_admin) + VALUES(?, ?, ?, ?, ?, ?, ?, ?) RETURNING * ` ) - .bind(id, PERSON, email, userKeyPair.pubKey, privkey, salt, JSON.stringify(properties)) + .bind(id, PERSON, email, userKeyPair.pubKey, privkey, salt, JSON.stringify(properties), admin ? 1 : null) .first() return personFromRow(row) diff --git a/frontend/mock-db/init.ts b/frontend/mock-db/init.ts index dc97a27..a9cde78 100644 --- a/frontend/mock-db/init.ts +++ b/frontend/mock-db/init.ts @@ -74,12 +74,21 @@ async function getOrCreatePerson( db: Database, { username, avatar, display_name }: Account ): Promise { - const person = await getPersonByEmail(db, username) + const isAdmin = username === 'george' + const email = `${username}@test.email` + const person = await getPersonByEmail(db, email) if (person) return person - const newPerson = await createPerson(domain, db, 'test-kek', username, { - icon: { url: avatar }, - name: display_name, - }) + const newPerson = await createPerson( + domain, + db, + 'test-kek', + email, + { + icon: { url: avatar }, + name: display_name, + }, + isAdmin + ) if (!newPerson) { throw new Error('Could not create Actor ' + username) } diff --git a/frontend/src/components/Settings/TextArea.tsx b/frontend/src/components/Settings/TextArea.tsx new file mode 100644 index 0000000..f83f302 --- /dev/null +++ b/frontend/src/components/Settings/TextArea.tsx @@ -0,0 +1,34 @@ +import { component$, useSignal } from '@builder.io/qwik' + +type Props = { + label: string + name?: string + description?: string + class?: string + invalid?: boolean + value?: string + required?: boolean +} + +export const TextArea = component$( + ({ class: className, label, name, description, invalid, value, required }) => { + const inputId = useSignal(`${label.replace(/\s+/g, '_')}___${crypto.randomUUID()}`).value + return ( +
+ + {!!description &&
{description}
} +