From 9c11b74c2ee8b099360c5d607ac50bd3bd13081c Mon Sep 17 00:00:00 2001 From: Sven Sauleau Date: Wed, 18 Jan 2023 13:31:50 +0000 Subject: [PATCH] MOW-118: switch from KV to DO for caching --- .github/workflows/deploy.yml | 34 ++++--- backend/src/cache/index.ts | 36 +++++++ backend/src/mastodon/notification.ts | 5 +- backend/src/mastodon/timeline.ts | 5 +- backend/src/types/env.ts | 3 +- backend/test/mastodon/notifications.spec.ts | 16 ++- backend/test/mastodon/statuses.spec.ts | 46 ++++----- backend/test/mastodon/timelines.spec.ts | 23 ++--- backend/test/utils.ts | 19 ++++ consumer/src/inbox.ts | 3 +- consumer/src/index.ts | 2 +- do/package.json | 14 +++ do/src/index.ts | 49 +++++++++ do/tsconfig.json | 106 ++++++++++++++++++++ do/wrangler.toml | 7 ++ frontend/mock-db/init.ts | 21 ++-- functions/api/v1/notifications.ts | 10 +- functions/api/v1/statuses.ts | 6 +- functions/api/v1/timelines/home.ts | 10 +- tf/main.tf | 23 ++++- 20 files changed, 340 insertions(+), 98 deletions(-) create mode 100644 backend/src/cache/index.ts create mode 100644 do/package.json create mode 100644 do/src/index.ts create mode 100644 do/tsconfig.json create mode 100644 do/wrangler.toml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6eb00b6..367bb77 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -166,6 +166,20 @@ jobs: echo "VAPID keys generated" fi + - name: Publish DO + uses: cloudflare/wrangler-action@2.0.0 + with: + apiToken: ${{ secrets.CF_API_TOKEN }} + command: publish --config do/wrangler.toml + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }} + + - name: Retrieve DO namespace + run: | + curl https://api.cloudflare.com/client/v4/accounts/${{ secrets.CF_ACCOUNT_ID }}/workers/durable_objects/namespaces \ + -H 'Authorization: Bearer ${{ secrets.CF_API_TOKEN }}' \ + | jq -r '.result[] | select( .script == "wildebeest-do" ) | .id' | awk '{print "do_cache_id="$1}' >> $GITHUB_ENV + - name: Configure run: terraform plan && terraform apply -auto-approve continue-on-error: true @@ -177,6 +191,7 @@ jobs: TF_VAR_cloudflare_deploy_domain: ${{ vars.CF_DEPLOY_DOMAIN }} TF_VAR_gh_username: ${{ env.OWNER_LOWER }} TF_VAR_d1_id: ${{ env.d1_id }} + TF_VAR_do_cache_id: ${{ env.do_cache_id }} TF_VAR_access_auth_domain: ${{ env.auth_domain }} TF_VAR_wd_instance_title: ${{ vars.INSTANCE_TITLE }} TF_VAR_wd_admin_email: ${{ vars.ADMIN_EMAIL }} @@ -214,18 +229,6 @@ jobs: env: CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }} - - name: retrieve Wildebeest cache KV namespace - uses: cloudflare/wrangler-action@2.0.0 - with: - command: kv:namespace list | jq -r '.[] | select( .title == "wildebeest-${{ env.OWNER_LOWER }}-cache" ) | .id' | awk '{print "cache_kv="$1}' >> $GITHUB_ENV - apiToken: ${{ secrets.CF_API_TOKEN }} - preCommands: | - echo "*** pre commands ***" - apt-get update && apt-get -y install jq - echo "******" - env: - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }} - - name: Create Queue uses: cloudflare/wrangler-action@2.0.0 with: @@ -243,9 +246,10 @@ jobs: echo "*** pre commands ***" echo -e "[[d1_databases]]\nbinding=\"DATABASE\"\ndatabase_name=\"wildebeest-${{ env.OWNER_LOWER }}\"\ndatabase_id=\"${{ env.d1_id }}\"\n" >> consumer/wrangler.toml - echo -e "[[kv_namespaces]]\n" >> consumer/wrangler.toml - echo -e "binding=\"KV_CACHE\"\n" >> consumer/wrangler.toml - echo -e "id=\"${{ env.cache_kv }}\"\n" >> consumer/wrangler.toml + echo -e "[durable_objects]\n" >> consumer/wrangler.toml + echo -e "bindings=[" >> consumer/wrangler.toml + echo -e "{name=\"DO_CACHE\",class_name=\"WildebeestCache\",script_name=\"wildebeest-do\"}," >> consumer/wrangler.toml + echo -e "]" >> consumer/wrangler.toml echo -e "[vars]\n" >> consumer/wrangler.toml echo -e "DOMAIN=\"${{ vars.CF_DEPLOY_DOMAIN }}\"\n" >> consumer/wrangler.toml diff --git a/backend/src/cache/index.ts b/backend/src/cache/index.ts new file mode 100644 index 0000000..bd136d5 --- /dev/null +++ b/backend/src/cache/index.ts @@ -0,0 +1,36 @@ +const CACHE_DO_NAME = 'cachev1' + +export interface Cache { + get(key: string): Promise + put(key: string, value: T): Promise +} + +export function cacheFromEnv(env: any): Cache { + return { + async get(key: string): Promise { + const id = env.DO_CACHE.idFromName(CACHE_DO_NAME) + const stub = env.DO_CACHE.get(id) + + const res = await stub.fetch('http://cache/' + key) + if (!res.ok) { + throw new Error(`DO cache returned ${res.status}: ${await res.text()}`) + } + + return (await res.json()) as T + }, + + async put(key: string, value: T): Promise { + const id = env.DO_CACHE.idFromName(CACHE_DO_NAME) + const stub = env.DO_CACHE.get(id) + + const res = await stub.fetch('http://cache/', { + method: 'PUT', + body: JSON.stringify({ key, value }), + }) + + if (!res.ok) { + throw new Error(`DO cache returned ${res.status}: ${await res.text()}`) + } + }, + } +} diff --git a/backend/src/mastodon/notification.ts b/backend/src/mastodon/notification.ts index 6ec5feb..3d6a8ac 100644 --- a/backend/src/mastodon/notification.ts +++ b/backend/src/mastodon/notification.ts @@ -10,6 +10,7 @@ import { WebPushResult } from 'wildebeest/backend/src/webpush/webpushinfos' import type { Actor } from 'wildebeest/backend/src/activitypub/actors' import type { NotificationType, Notification } from 'wildebeest/backend/src/types/notification' import { getSubscriptionForAllClients } from 'wildebeest/backend/src/mastodon/subscription' +import type { Cache } from 'wildebeest/backend/src/cache' export async function createNotification( db: D1Database, @@ -250,7 +251,7 @@ export async function getNotifications(db: D1Database, actor: Actor, domain: str return out } -export async function pregenerateNotifications(db: D1Database, cache: KVNamespace, actor: Actor, domain: string) { +export async function pregenerateNotifications(db: D1Database, cache: Cache, actor: Actor, domain: string) { const notifications = await getNotifications(db, actor, domain) - await cache.put(actor.id + '/notifications', JSON.stringify(notifications)) + await cache.put(actor.id + '/notifications', notifications) } diff --git a/backend/src/mastodon/timeline.ts b/backend/src/mastodon/timeline.ts index c735409..1d287e8 100644 --- a/backend/src/mastodon/timeline.ts +++ b/backend/src/mastodon/timeline.ts @@ -3,10 +3,11 @@ import { getFollowingId } from 'wildebeest/backend/src/mastodon/follow' import type { Actor } from 'wildebeest/backend/src/activitypub/actors/' import { toMastodonStatusFromRow } from './status' import { PUBLIC_GROUP } from 'wildebeest/backend/src/activitypub/activities' +import type { Cache } from 'wildebeest/backend/src/cache' -export async function pregenerateTimelines(domain: string, db: D1Database, cache: KVNamespace, actor: Actor) { +export async function pregenerateTimelines(domain: string, db: D1Database, cache: Cache, actor: Actor) { const timeline = await getHomeTimeline(domain, db, actor) - await cache.put(actor.id + '/timeline/home', JSON.stringify(timeline)) + await cache.put(actor.id + '/timeline/home', timeline) } export async function getHomeTimeline(domain: string, db: D1Database, actor: Actor): Promise> { diff --git a/backend/src/types/env.ts b/backend/src/types/env.ts index ee1b9cd..f913309 100644 --- a/backend/src/types/env.ts +++ b/backend/src/types/env.ts @@ -2,9 +2,10 @@ import type { Queue, MessageBody } from 'wildebeest/backend/src/types/queue' export interface Env { DATABASE: D1Database - KV_CACHE: KVNamespace + // FIXME: shouldn't it be USER_KEY? userKEK: string QUEUE: Queue + DO_CACHE: DurableObjectNamespace CF_ACCOUNT_ID: string CF_API_TOKEN: string diff --git a/backend/test/mastodon/notifications.spec.ts b/backend/test/mastodon/notifications.spec.ts index 9bcc53e..0b9647a 100644 --- a/backend/test/mastodon/notifications.spec.ts +++ b/backend/test/mastodon/notifications.spec.ts @@ -4,7 +4,7 @@ import { createPublicNote } from 'wildebeest/backend/src/activitypub/objects/not import { createNotification, insertFollowNotification } from 'wildebeest/backend/src/mastodon/notification' import { createPerson } from 'wildebeest/backend/src/activitypub/actors' import * as notifications from 'wildebeest/functions/api/v1/notifications' -import { makeDB, assertJSON, createTestClient } from '../utils' +import { makeCache, makeDB, assertJSON, createTestClient } from '../utils' import { strict as assert } from 'node:assert/strict' import { sendLikeNotification } from 'wildebeest/backend/src/mastodon/notification' import { createSubscription } from 'wildebeest/backend/src/mastodon/subscription' @@ -32,15 +32,13 @@ describe('Mastodon APIs', () => { test('returns notifications stored in KV cache', async () => { const db = await makeDB() const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') - const kv_cache: any = { - async get(key: string) { - assert.equal(key, connectedActor.id + '/notifications') - return 'cached data' - }, - } + const cache = makeCache() + + await cache.put(connectedActor.id + '/notifications', 12345) + const req = new Request('https://' + domain) - const data = await notifications.handleRequest(req, kv_cache, connectedActor) - assert.equal(await data.text(), 'cached data') + const data = await notifications.handleRequest(req, cache, connectedActor) + assert.equal(await data.json(), 12345) }) test('returns notifications stored in db', async () => { diff --git a/backend/test/mastodon/statuses.spec.ts b/backend/test/mastodon/statuses.spec.ts index 303d2e4..1d2e83f 100644 --- a/backend/test/mastodon/statuses.spec.ts +++ b/backend/test/mastodon/statuses.spec.ts @@ -12,7 +12,7 @@ import * as statuses_context from 'wildebeest/functions/api/v1/statuses/[id]/con import { createPerson } from 'wildebeest/backend/src/activitypub/actors' import { insertLike } from 'wildebeest/backend/src/mastodon/like' import { insertReblog } from 'wildebeest/backend/src/mastodon/reblog' -import { isUrlValid, makeDB, assertJSON, streamToArrayBuffer, makeQueue } from '../utils' +import { isUrlValid, makeDB, assertJSON, streamToArrayBuffer, makeQueue, makeCache } from '../utils' import * as activities from 'wildebeest/backend/src/activitypub/activities' import { addFollowing, acceptFollowing } from 'wildebeest/backend/src/mastodon/follow' import { MessageType } from 'wildebeest/backend/src/types/queue' @@ -20,9 +20,7 @@ import { MessageType } from 'wildebeest/backend/src/types/queue' const userKEK = 'test_kek4' const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) const domain = 'cloudflare.com' -const kv_cache: any = { - async put() {}, -} +const cache = makeCache() describe('Mastodon APIs', () => { describe('statuses', () => { @@ -38,7 +36,7 @@ describe('Mastodon APIs', () => { }) const connectedActor: any = {} - const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache) assert.equal(res.status, 400) }) @@ -58,7 +56,7 @@ describe('Mastodon APIs', () => { }) const connectedActor = actor - const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache) assert.equal(res.status, 200) assertJSON(res) @@ -96,14 +94,7 @@ describe('Mastodon APIs', () => { const db = await makeDB() const queue = makeQueue() const actor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') - - let cache = null - const kv_cache: any = { - async put(key: string, value: any) { - assert.equal(key, actor.id + '/timeline/home') - cache = value - }, - } + const cache = makeCache() const body = { status: 'my status', @@ -115,16 +106,17 @@ describe('Mastodon APIs', () => { body: JSON.stringify(body), }) - const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache) assert.equal(res.status, 200) assertJSON(res) const data = await res.json() - assert(cache) - const timeline = JSON.parse(cache) - assert.equal(timeline.length, 1) - assert.equal(timeline[0].id, data.id) + const cachedData = await cache.get(actor.id + '/timeline/home') + console.log({ cachedData }) + assert(cachedData) + assert.equal(cachedData.length, 1) + assert.equal(cachedData[0].id, data.id) }) test("create new status adds to Actor's outbox", async () => { @@ -143,7 +135,7 @@ describe('Mastodon APIs', () => { }) const connectedActor = actor - const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache) assert.equal(res.status, 200) const row = await db.prepare(`SELECT count(*) as count FROM outbox_objects`).first() @@ -171,7 +163,7 @@ describe('Mastodon APIs', () => { body: JSON.stringify(body), }) - const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache) assert.equal(res.status, 200) assert.equal(queue.messages.length, 2) @@ -250,7 +242,7 @@ describe('Mastodon APIs', () => { }) const connectedActor = actor - const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache) assert.equal(res.status, 200) assert(deliveredNote) @@ -281,7 +273,7 @@ describe('Mastodon APIs', () => { body: JSON.stringify(body), }) - const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, connectedActor, userKEK, queue, cache) assert.equal(res.status, 200) const data = await res.json() @@ -560,7 +552,7 @@ describe('Mastodon APIs', () => { body: JSON.stringify(body), }) - const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache) assert.equal(res.status, 404) }) @@ -581,7 +573,7 @@ describe('Mastodon APIs', () => { body: JSON.stringify(body), }) - const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache) assert.equal(res.status, 200) const data = await res.json() @@ -625,7 +617,7 @@ describe('Mastodon APIs', () => { body: JSON.stringify(body), }) - const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache) assert.equal(res.status, 400) const data = await res.json() assert(data.error.includes('Limit exceeded')) @@ -650,7 +642,7 @@ describe('Mastodon APIs', () => { body, }) - const res = await statuses.handleRequest(req, db, actor, userKEK, queue, kv_cache) + const res = await statuses.handleRequest(req, db, actor, userKEK, queue, cache) assert.equal(res.status, 400) const data = await res.json() assert(data.error.includes('Limit exceeded')) diff --git a/backend/test/mastodon/timelines.spec.ts b/backend/test/mastodon/timelines.spec.ts index 433c921..f9ac301 100644 --- a/backend/test/mastodon/timelines.spec.ts +++ b/backend/test/mastodon/timelines.spec.ts @@ -5,7 +5,7 @@ import { addFollowing, acceptFollowing } from 'wildebeest/backend/src/mastodon/f import { createPublicNote, createPrivateNote } from 'wildebeest/backend/src/activitypub/objects/note' import { addObjectInOutbox } from 'wildebeest/backend/src/activitypub/actors/outbox' import { createPerson } from 'wildebeest/backend/src/activitypub/actors' -import { makeDB, assertCORS, assertJSON } from '../utils' +import { makeDB, assertCORS, assertJSON, makeCache } from '../utils' import * as timelines_home from 'wildebeest/functions/api/v1/timelines/home' import * as timelines_public from 'wildebeest/functions/api/v1/timelines/public' import * as timelines from 'wildebeest/backend/src/mastodon/timeline' @@ -106,27 +106,20 @@ describe('Mastodon APIs', () => { test('home returns cache', async () => { const db = await makeDB() const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') - const kv_cache: any = { - async get(key: string) { - assert.equal(key, connectedActor.id + '/timeline/home') - return 'cached data' - }, - } + const cache = makeCache() + await cache.put(connectedActor.id + '/timeline/home', 12345) + const req = new Request('https://' + domain) - const data = await timelines_home.handleRequest(req, kv_cache, connectedActor) - assert.equal(await data.text(), 'cached data') + const data = await timelines_home.handleRequest(req, cache, connectedActor) + assert.equal(await data.json(), 12345) }) test('home returns empty if not in cache', async () => { const db = await makeDB() const connectedActor = await createPerson(domain, db, userKEK, 'sven@cloudflare.com') - const kv_cache: any = { - async get() { - return null - }, - } + const cache = makeCache() const req = new Request('https://' + domain) - const data = await timelines_home.handleRequest(req, kv_cache, connectedActor) + const data = await timelines_home.handleRequest(req, cache, connectedActor) const posts = await data.json>() assert.equal(posts.length, 0) diff --git a/backend/test/utils.ts b/backend/test/utils.ts index 47b5bb7..6159ab0 100644 --- a/backend/test/utils.ts +++ b/backend/test/utils.ts @@ -1,4 +1,5 @@ import { strict as assert } from 'node:assert/strict' +import type { Cache } from 'wildebeest/backend/src/cache' import type { Queue } from 'wildebeest/backend/src/types/queue' import { createClient } from 'wildebeest/backend/src/mastodon/client' import type { Client } from 'wildebeest/backend/src/mastodon/client' @@ -91,6 +92,24 @@ export function makeQueue(): TestQueue { } } +export function makeCache(): Cache { + const cache: any = {} + + return { + async get(key: string): Promise { + if (cache[key]) { + return cache[key] as T + } else { + return null + } + }, + + async put(key: string, value: T): Promise { + cache[key] = value + }, + } +} + export function isUUID(v: string): boolean { assert.equal(typeof v, 'string') if (v.match('^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') === null) { diff --git a/consumer/src/inbox.ts b/consumer/src/inbox.ts index 988a95c..3801f16 100644 --- a/consumer/src/inbox.ts +++ b/consumer/src/inbox.ts @@ -2,6 +2,7 @@ import type { MessageBody, InboxMessageBody } from 'wildebeest/backend/src/types import * as activityHandler from 'wildebeest/backend/src/activitypub/activities/handle' import * as notification from 'wildebeest/backend/src/mastodon/notification' import * as timeline from 'wildebeest/backend/src/mastodon/timeline' +import { cacheFromEnv } from 'wildebeest/backend/src/cache' import type { Actor } from 'wildebeest/backend/src/activitypub/actors' import type { Env } from './' @@ -9,7 +10,7 @@ export async function handleInboxMessage(env: Env, actor: Actor, message: InboxM const domain = env.DOMAIN const db = env.DATABASE const adminEmail = env.ADMIN_EMAIL - const cache = env.KV_CACHE + const cache = cacheFromEnv(env) const activity = message.activity await activityHandler.handle(domain, activity, db, message.userKEK, adminEmail, message.vapidKeys) diff --git a/consumer/src/index.ts b/consumer/src/index.ts index b4fb5cb..5c94115 100644 --- a/consumer/src/index.ts +++ b/consumer/src/index.ts @@ -12,7 +12,7 @@ export type Env = { DATABASE: D1Database DOMAIN: string ADMIN_EMAIL: string - KV_CACHE: KVNamespace + DO_CACHE: DurableObjectNamespace } export default { diff --git a/do/package.json b/do/package.json new file mode 100644 index 0000000..e26b4ee --- /dev/null +++ b/do/package.json @@ -0,0 +1,14 @@ +{ + "name": "wildebeest-do", + "version": "0.0.0", + "devDependencies": { + "@cloudflare/workers-types": "^4.20221111.1", + "typescript": "^4.9.4", + "wrangler": "2.7.1" + }, + "private": true, + "scripts": { + "start": "wrangler dev", + "deploy": "wrangler publish" + } +} diff --git a/do/src/index.ts b/do/src/index.ts new file mode 100644 index 0000000..b905b19 --- /dev/null +++ b/do/src/index.ts @@ -0,0 +1,49 @@ +export interface Env { + DO: DurableObjectNamespace +} + +export default { + async fetch(request: Request, env: Env) { + try { + const id = env.DO.idFromName('default') + const obj = env.DO.get(id) + return obj.fetch(request) + } catch (err: any) { + return new Response(err.stack, { status: 500 }) + } + }, +} + +export class WildebeestCache { + private storage: DurableObjectStorage + + constructor(state: DurableObjectState) { + this.storage = state.storage + } + + async fetch(request: Request) { + if (request.method === 'GET') { + const { pathname } = new URL(request.url) + const key = pathname.slice(1) // remove the leading slash from path + + const value = await this.storage.get(key) + if (value === undefined) { + console.log(`Get ${key} MISS`) + return new Response('', { status: 404 }) + } + + console.log(`Get ${key} HIT`) + return new Response(JSON.stringify(value)) + } + + if (request.method === 'PUT') { + const { key, value } = await request.json() + console.log(`Set ${key}`) + + await this.storage.put(key, value) + return new Response('', { status: 201 }) + } + + return new Response('', { status: 400 }) + } +} diff --git a/do/tsconfig.json b/do/tsconfig.json new file mode 100644 index 0000000..d8a199c --- /dev/null +++ b/do/tsconfig.json @@ -0,0 +1,106 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Projects */ + // "incremental": true, /* Enable incremental compilation */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + "lib": [ + "es2021" + ] /* Specify a set of bundled library declaration files that describe the target runtime environment. */, + "jsx": "react" /* Specify what JSX code is generated. */, + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ + // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + + /* Modules */ + "module": "es2022" /* Specify what module code is generated. */, + // "rootDir": "./", /* Specify the root folder within your source files. */ + "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ + "types": [ + "@cloudflare/workers-types", + "jest" + ] /* Specify type package names to be included without being referenced in a source file. */, + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + "resolveJsonModule": true /* Enable importing .json files */, + // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */, + "checkJs": false /* Enable error reporting in type-checked JavaScript files. */, + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + "noEmit": true /* Disable emitting files from a compilation. */, + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */, + "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, + // "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, + + /* Type Checking */ + "strict": true /* Enable all strict type-checking options. */, + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ + // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ + // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} diff --git a/do/wrangler.toml b/do/wrangler.toml new file mode 100644 index 0000000..6ce553f --- /dev/null +++ b/do/wrangler.toml @@ -0,0 +1,7 @@ +name = "wildebeest-do" +main = "src/index.ts" +compatibility_date = "2023-01-18" + +[[migrations]] +tag = "v1" +new_classes = ["WildebeestCache"] diff --git a/frontend/mock-db/init.ts b/frontend/mock-db/init.ts index 22f98f4..c938fa9 100644 --- a/frontend/mock-db/init.ts +++ b/frontend/mock-db/init.ts @@ -6,14 +6,15 @@ import type { Account, MastodonStatus } from 'wildebeest/frontend/src/types' const kek = 'test-kek' /* eslint-disable @typescript-eslint/no-empty-function */ -const queue = { - async send() {}, - async sendBatch() {}, +const queue: unknown = { + send() {}, + sendBatch() {}, } -const kv_cache = { - async put() {}, +/* eslint-disable @typescript-eslint/no-empty-function */ +const cache: unknown = { + get() {}, + put() {}, } -/* eslint-enable @typescript-eslint/no-empty-function */ /** * Run helper commands to initialize the database with actors, statuses, etc. @@ -30,7 +31,7 @@ export async function init(domain: string, db: D1Database) { const reblogger = await getOrCreatePerson(domain, db, rebloggerAccount) // Reblog an arbitrary status with this reblogger const statusToReblog = loadedStatuses[2] - await reblogStatus(db, reblogger, statusToReblog) + await reblogStatus(db, reblogger, statusToReblog, domain) } /** @@ -49,7 +50,7 @@ async function createStatus(db: D1Database, actor: Person, status: string, visib headers, body: JSON.stringify(body), }) - const resp = await statusesAPI.handleRequest(req, db, actor, kek, queue, kv_cache as unknown as KVNamespace) + const resp = await statusesAPI.handleRequest(req, db, actor, kek, queue, cache) return (await resp.json()) as MastodonStatus } @@ -70,6 +71,6 @@ async function getOrCreatePerson( return newPerson } -async function reblogStatus(db: D1Database, actor: Person, status: MastodonStatus) { - await reblogAPI.handleRequest(db, status.id, actor, kek, queue) +async function reblogStatus(db: D1Database, actor: Person, status: MastodonStatus, domain: string) { + await reblogAPI.handleRequest(db, status.id, actor, kek, queue, domain) } diff --git a/functions/api/v1/notifications.ts b/functions/api/v1/notifications.ts index 06c5f43..f4422df 100644 --- a/functions/api/v1/notifications.ts +++ b/functions/api/v1/notifications.ts @@ -4,9 +4,11 @@ import { cors } from 'wildebeest/backend/src/utils/cors' import type { Env } from 'wildebeest/backend/src/types/env' import type { Person } from 'wildebeest/backend/src/activitypub/actors' import type { ContextData } from 'wildebeest/backend/src/types/context' +import type { Cache } from 'wildebeest/backend/src/cache' +import { cacheFromEnv } from 'wildebeest/backend/src/cache' export const onRequest: PagesFunction = async ({ request, env, data }) => { - return handleRequest(request, env.KV_CACHE, data.connectedActor) + return handleRequest(request, cacheFromEnv(env), data.connectedActor) } const headers = { @@ -14,7 +16,7 @@ const headers = { 'content-type': 'application/json; charset=utf-8', } -export async function handleRequest(request: Request, cache: KVNamespace, connectedActor: Person): Promise { +export async function handleRequest(request: Request, cache: Cache, connectedActor: Person): Promise { const url = new URL(request.url) if (url.searchParams.has('max_id')) { // We just return the pregenerated notifications, without any filter for @@ -23,9 +25,9 @@ export async function handleRequest(request: Request, cache: KVNamespace, connec return new Response(JSON.stringify([]), { headers }) } - const notifications = await cache.get(connectedActor.id + '/notifications') + const notifications = await cache.get(connectedActor.id + '/notifications') if (notifications === null) { return new Response(JSON.stringify([]), { headers }) } - return new Response(notifications, { headers }) + return new Response(JSON.stringify(notifications), { headers }) } diff --git a/functions/api/v1/statuses.ts b/functions/api/v1/statuses.ts index d1ca748..a5ab72a 100644 --- a/functions/api/v1/statuses.ts +++ b/functions/api/v1/statuses.ts @@ -21,6 +21,8 @@ import { readBody } from 'wildebeest/backend/src/utils/body' import * as errors from 'wildebeest/backend/src/errors' import type { Visibility } from 'wildebeest/backend/src/types' import { toMastodonStatusFromObject } from 'wildebeest/backend/src/mastodon/status' +import type { Cache } from 'wildebeest/backend/src/cache' +import { cacheFromEnv } from 'wildebeest/backend/src/cache' type StatusCreate = { status: string @@ -31,7 +33,7 @@ type StatusCreate = { } export const onRequest: PagesFunction = async ({ request, env, data }) => { - return handleRequest(request, env.DATABASE, data.connectedActor, env.userKEK, env.QUEUE, env.KV_CACHE) + return handleRequest(request, env.DATABASE, data.connectedActor, env.userKEK, env.QUEUE, cacheFromEnv(env)) } // FIXME: add tests for delivery to followers and mentions to a specific Actor. @@ -41,7 +43,7 @@ export async function handleRequest( connectedActor: Person, userKEK: string, queue: Queue, - cache: KVNamespace + cache: Cache ): Promise { // TODO: implement Idempotency-Key diff --git a/functions/api/v1/timelines/home.ts b/functions/api/v1/timelines/home.ts index 1c9c895..e3b6b69 100644 --- a/functions/api/v1/timelines/home.ts +++ b/functions/api/v1/timelines/home.ts @@ -2,6 +2,8 @@ import type { Env } from 'wildebeest/backend/src/types/env' import { cors } from 'wildebeest/backend/src/utils/cors' import type { ContextData } from 'wildebeest/backend/src/types/context' import type { Actor } from 'wildebeest/backend/src/activitypub/actors/' +import type { Cache } from 'wildebeest/backend/src/cache' +import { cacheFromEnv } from 'wildebeest/backend/src/cache' const headers = { ...cors(), @@ -9,10 +11,10 @@ const headers = { } export const onRequest: PagesFunction = async ({ request, env, data }) => { - return handleRequest(request, env.KV_CACHE, data.connectedActor) + return handleRequest(request, cacheFromEnv(env), data.connectedActor) } -export async function handleRequest(request: Request, cache: KVNamespace, actor: Actor): Promise { +export async function handleRequest(request: Request, cache: Cache, actor: Actor): Promise { const url = new URL(request.url) if (url.searchParams.has('max_id')) { // We just return the pregenerated notifications, without any filter for @@ -21,9 +23,9 @@ export async function handleRequest(request: Request, cache: KVNamespace, actor: return new Response(JSON.stringify([]), { headers }) } - const timeline = await cache.get(actor.id + '/timeline/home') + const timeline = await cache.get(actor.id + '/timeline/home') if (timeline === null) { return new Response(JSON.stringify([]), { headers }) } - return new Response(timeline, { headers }) + return new Response(JSON.stringify(timeline), { headers }) } diff --git a/tf/main.tf b/tf/main.tf index bf593b4..4783a86 100644 --- a/tf/main.tf +++ b/tf/main.tf @@ -27,6 +27,11 @@ variable "d1_id" { sensitive = true } +variable "do_cache_id" { + type = string + sensitive = true +} + variable "access_auth_domain" { type = string sensitive = true @@ -76,9 +81,11 @@ provider "cloudflare" { api_token = var.cloudflare_api_token } -resource "cloudflare_workers_kv_namespace" "wildebeest_cache" { - account_id = var.cloudflare_account_id - title = "wildebeest-${lower(var.gh_username)}-cache" +// The KV cache namespace isn't used anymore but Terraform isn't able +// to remove the binding from the Pages project, so leaving for now. +resource "cloudflare_workers_kv_namespace" "wildebeest_cache" { + account_id = var.cloudflare_account_id + title = "wildebeest-${lower(var.gh_username)}-cache" } resource "cloudflare_workers_kv_namespace" "terraform_state" { @@ -117,13 +124,19 @@ resource "cloudflare_pages_project" "wildebeest_pages_project" { SENTRY_ACCESS_CLIENT_ID = var.sentry_access_client_id SENTRY_ACCESS_CLIENT_SECRET = var.sentry_access_client_secret } - kv_namespaces = { - KV_CACHE = sensitive(cloudflare_workers_kv_namespace.wildebeest_cache.id) + + kv_namespaces = { + KV_CACHE = sensitive(cloudflare_workers_kv_namespace.wildebeest_cache.id) } + d1_databases = { DATABASE = sensitive(var.d1_id) } + durable_object_namespaces = { + DO_CACHE = sensitive(var.do_cache_id) + } + compatibility_date = "2023-01-09" } }